diff --git a/LTA_openwebtext_dualt/logs/ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m_20260504_195806.log b/LTA_openwebtext_dualt/logs/ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m_20260504_195806.log new file mode 100644 index 0000000000000000000000000000000000000000..47c720858717a261203490eb77e3a14f550ead59 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m_20260504_195806.log @@ -0,0 +1,305 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +NCCL version 2.25.1+cuda12.8 +{ + "task": "ar_lm", + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "wrapped_streaming", + "vocab_size": 30522, + "bos_id": 101, + "eos_id": 102, + "save_dir": "runs/ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m_20260504_195806", + "params": 108440832, + "batch_size": 64, + "grad_accum": 2, + "effective_batch_size": 512, + "global_batch_size": 512, + "max_len": 128, + "wrap": true, + "text_detokenizer": "lm1b", + "openwebtext_split": "all", + "torch_compile": false +} +step=20 micro_steps=40 elapsed=4.3s lr=2.520000e-06 loss=10.3798 ppl=32478.6943 acc=0.0027 tokens=8128.0000 +step=40 micro_steps=80 elapsed=3.6s lr=4.920000e-06 loss=9.6982 ppl=16677.9630 acc=0.0420 tokens=8128.0000 +step=60 micro_steps=120 elapsed=3.6s lr=7.320000e-06 loss=9.1128 ppl=9132.2660 acc=0.0511 tokens=8128.0000 +step=80 micro_steps=160 elapsed=3.6s lr=9.720000e-06 loss=8.8143 ppl=6750.1859 acc=0.0650 tokens=8128.0000 +step=100 micro_steps=200 elapsed=3.6s lr=1.212000e-05 loss=8.5221 ppl=5041.4905 acc=0.0799 tokens=8128.0000 +step=120 micro_steps=240 elapsed=3.6s lr=1.452000e-05 loss=8.2581 ppl=3870.1153 acc=0.0867 tokens=8128.0000 +step=140 micro_steps=280 elapsed=3.6s lr=1.692000e-05 loss=7.9724 ppl=2911.8598 acc=0.0919 tokens=8128.0000 +step=160 micro_steps=320 elapsed=3.6s lr=1.932000e-05 loss=7.6568 ppl=2123.6313 acc=0.1067 tokens=8128.0000 +step=180 micro_steps=360 elapsed=3.6s lr=2.172000e-05 loss=7.3502 ppl=1563.0248 acc=0.1163 tokens=8128.0000 +step=200 micro_steps=400 elapsed=3.6s lr=2.412000e-05 loss=7.0837 ppl=1195.9628 acc=0.1251 tokens=8128.0000 +step=220 micro_steps=440 elapsed=3.6s lr=2.652000e-05 loss=6.8824 ppl=976.4170 acc=0.1333 tokens=8128.0000 +step=240 micro_steps=480 elapsed=3.6s lr=2.892000e-05 loss=6.7489 ppl=854.0937 acc=0.1375 tokens=8128.0000 +step=260 micro_steps=520 elapsed=3.6s lr=3.132000e-05 loss=6.6435 ppl=768.6732 acc=0.1441 tokens=8128.0000 +step=280 micro_steps=560 elapsed=3.6s lr=3.372000e-05 loss=6.5504 ppl=700.1528 acc=0.1519 tokens=8128.0000 +step=300 micro_steps=600 elapsed=3.6s lr=3.612000e-05 loss=6.4669 ppl=644.4497 acc=0.1587 tokens=8128.0000 +step=320 micro_steps=640 elapsed=3.6s lr=3.852000e-05 loss=6.3869 ppl=594.5753 acc=0.1655 tokens=8128.0000 +step=340 micro_steps=680 elapsed=3.6s lr=4.092000e-05 loss=6.2992 ppl=544.6934 acc=0.1715 tokens=8128.0000 +step=360 micro_steps=720 elapsed=3.6s lr=4.332000e-05 loss=6.2257 ppl=506.2021 acc=0.1749 tokens=8128.0000 +step=380 micro_steps=760 elapsed=3.6s lr=4.572000e-05 loss=6.1530 ppl=470.6410 acc=0.1788 tokens=8128.0000 +step=400 micro_steps=800 elapsed=3.6s lr=4.812000e-05 loss=6.0747 ppl=435.1815 acc=0.1824 tokens=8128.0000 +step=420 micro_steps=840 elapsed=3.6s lr=5.052000e-05 loss=5.9999 ppl=403.7083 acc=0.1866 tokens=8128.0000 +step=440 micro_steps=880 elapsed=3.6s lr=5.292000e-05 loss=5.9622 ppl=388.8123 acc=0.1894 tokens=8128.0000 +step=460 micro_steps=920 elapsed=3.6s lr=5.532000e-05 loss=5.9080 ppl=368.1609 acc=0.1923 tokens=8128.0000 +step=480 micro_steps=960 elapsed=3.6s lr=5.772000e-05 loss=5.8534 ppl=348.7774 acc=0.1953 tokens=8128.0000 +step=500 micro_steps=1000 elapsed=3.6s lr=6.012000e-05 loss=5.8124 ppl=334.7449 acc=0.1972 tokens=8128.0000 +step=520 micro_steps=1040 elapsed=3.6s lr=6.252000e-05 loss=5.7628 ppl=318.4989 acc=0.1996 tokens=8128.0000 +step=540 micro_steps=1080 elapsed=3.6s lr=6.492000e-05 loss=5.7351 ppl=309.8943 acc=0.2010 tokens=8128.0000 +step=560 micro_steps=1120 elapsed=3.6s lr=6.732000e-05 loss=5.6905 ppl=296.3728 acc=0.2039 tokens=8128.0000 +step=580 micro_steps=1160 elapsed=3.6s lr=6.972000e-05 loss=5.6509 ppl=284.9865 acc=0.2069 tokens=8128.0000 +step=600 micro_steps=1200 elapsed=3.6s lr=7.212000e-05 loss=5.6142 ppl=274.5673 acc=0.2090 tokens=8128.0000 +step=620 micro_steps=1240 elapsed=3.6s lr=7.452000e-05 loss=5.5644 ppl=261.2549 acc=0.2113 tokens=8128.0000 +step=640 micro_steps=1280 elapsed=3.6s lr=7.692000e-05 loss=5.5398 ppl=254.9341 acc=0.2136 tokens=8128.0000 +step=660 micro_steps=1320 elapsed=3.6s lr=7.932000e-05 loss=5.5199 ppl=249.8425 acc=0.2146 tokens=8128.0000 +step=680 micro_steps=1360 elapsed=3.6s lr=8.172000e-05 loss=5.4786 ppl=239.7250 acc=0.2176 tokens=8128.0000 +step=700 micro_steps=1400 elapsed=3.6s lr=8.412000e-05 loss=5.4594 ppl=235.2477 acc=0.2177 tokens=8128.0000 +step=720 micro_steps=1440 elapsed=3.6s lr=8.652000e-05 loss=5.4230 ppl=226.7282 acc=0.2195 tokens=8128.0000 +step=740 micro_steps=1480 elapsed=3.6s lr=8.892000e-05 loss=5.3881 ppl=219.0196 acc=0.2224 tokens=8128.0000 +step=760 micro_steps=1520 elapsed=3.6s lr=9.132000e-05 loss=5.3681 ppl=214.6676 acc=0.2242 tokens=8128.0000 +step=780 micro_steps=1560 elapsed=3.6s lr=9.372000e-05 loss=5.3561 ppl=212.1429 acc=0.2236 tokens=8128.0000 +step=800 micro_steps=1600 elapsed=3.6s lr=9.612000e-05 loss=5.3154 ppl=203.6174 acc=0.2266 tokens=8128.0000 +step=820 micro_steps=1640 elapsed=3.6s lr=9.852000e-05 loss=5.2852 ppl=197.5881 acc=0.2281 tokens=8128.0000 +step=840 micro_steps=1680 elapsed=3.6s lr=1.009200e-04 loss=5.2622 ppl=193.1013 acc=0.2307 tokens=8128.0000 +step=860 micro_steps=1720 elapsed=3.6s lr=1.033200e-04 loss=5.2473 ppl=190.2556 acc=0.2309 tokens=8128.0000 +step=880 micro_steps=1760 elapsed=3.6s lr=1.057200e-04 loss=5.2202 ppl=185.1611 acc=0.2324 tokens=8128.0000 +step=900 micro_steps=1800 elapsed=3.6s lr=1.081200e-04 loss=5.1907 ppl=179.7384 acc=0.2351 tokens=8128.0000 +step=920 micro_steps=1840 elapsed=3.6s lr=1.105200e-04 loss=5.1668 ppl=175.4561 acc=0.2370 tokens=8128.0000 +step=940 micro_steps=1880 elapsed=3.6s lr=1.129200e-04 loss=5.1512 ppl=172.8847 acc=0.2380 tokens=8128.0000 +step=960 micro_steps=1920 elapsed=3.6s lr=1.153200e-04 loss=5.1224 ppl=167.9127 acc=0.2385 tokens=8128.0000 +step=980 micro_steps=1960 elapsed=3.6s lr=1.177200e-04 loss=5.1084 ppl=165.6339 acc=0.2393 tokens=8128.0000 +step=1000 micro_steps=2000 elapsed=3.6s lr=1.201200e-04 loss=5.0897 ppl=162.5289 acc=0.2409 tokens=8128.0000 +[sample step=1000] [CLS] bash turned on and had five - day splash design. [SEP] the rails of proficiency were eastern, with the drum - smashed races of tif hoc sabbath. [SEP] good could be in liechtenstein, but not 314 of the same floor is imminent. [SEP] kristin fox expired 24 - 15 on beth gregor marsh's celebration and he was volleyed with his being sacked as top - completing practice in oakland. [SEP] passion for poverty and cuts: chief executive the company's office reported that markedly of last year'slight and the administration's potentially derivatives force indicates to become careful friday. [SEP] overall, then the economic compensation program has acquired [SEP] +step=1020 micro_steps=2040 elapsed=6.4s lr=1.225200e-04 loss=5.0651 ppl=158.5900 acc=0.2426 tokens=8128.0000 +step=1040 micro_steps=2080 elapsed=3.6s lr=1.249200e-04 loss=5.0462 ppl=155.6245 acc=0.2436 tokens=8128.0000 +step=1060 micro_steps=2120 elapsed=3.6s lr=1.273200e-04 loss=5.0489 ppl=156.0155 acc=0.2436 tokens=8128.0000 +step=1080 micro_steps=2160 elapsed=3.6s lr=1.297200e-04 loss=5.0148 ppl=150.8239 acc=0.2466 tokens=8128.0000 +step=1100 micro_steps=2200 elapsed=3.6s lr=1.321200e-04 loss=4.9821 ppl=145.9810 acc=0.2484 tokens=8128.0000 +step=1120 micro_steps=2240 elapsed=3.6s lr=1.345200e-04 loss=4.9698 ppl=144.2456 acc=0.2497 tokens=8128.0000 +step=1140 micro_steps=2280 elapsed=3.6s lr=1.369200e-04 loss=4.9559 ppl=142.2155 acc=0.2496 tokens=8128.0000 +step=1160 micro_steps=2320 elapsed=3.6s lr=1.393200e-04 loss=4.9270 ppl=138.0649 acc=0.2513 tokens=8128.0000 +step=1180 micro_steps=2360 elapsed=3.6s lr=1.417200e-04 loss=4.9200 ppl=137.1738 acc=0.2521 tokens=8128.0000 +step=1200 micro_steps=2400 elapsed=3.6s lr=1.441200e-04 loss=4.9104 ppl=135.8119 acc=0.2530 tokens=8128.0000 +step=1220 micro_steps=2440 elapsed=3.6s lr=1.465200e-04 loss=4.8784 ppl=131.6055 acc=0.2556 tokens=8128.0000 +step=1240 micro_steps=2480 elapsed=3.6s lr=1.489200e-04 loss=4.8727 ppl=130.8100 acc=0.2568 tokens=8128.0000 +step=1260 micro_steps=2520 elapsed=3.6s lr=1.513200e-04 loss=4.8468 ppl=127.4586 acc=0.2564 tokens=8128.0000 +step=1280 micro_steps=2560 elapsed=3.6s lr=1.537200e-04 loss=4.8371 ppl=126.2536 acc=0.2580 tokens=8128.0000 +step=1300 micro_steps=2600 elapsed=3.6s lr=1.561200e-04 loss=4.8055 ppl=122.3074 acc=0.2601 tokens=8128.0000 +step=1320 micro_steps=2640 elapsed=3.6s lr=1.585200e-04 loss=4.8018 ppl=121.8846 acc=0.2605 tokens=8128.0000 +step=1340 micro_steps=2680 elapsed=3.6s lr=1.609200e-04 loss=4.7835 ppl=119.6142 acc=0.2619 tokens=8128.0000 +step=1360 micro_steps=2720 elapsed=3.6s lr=1.633200e-04 loss=4.7571 ppl=116.5595 acc=0.2649 tokens=8128.0000 +step=1380 micro_steps=2760 elapsed=3.6s lr=1.657200e-04 loss=4.7526 ppl=116.0234 acc=0.2641 tokens=8128.0000 +step=1400 micro_steps=2800 elapsed=3.6s lr=1.681200e-04 loss=4.7498 ppl=115.6697 acc=0.2641 tokens=8128.0000 +step=1420 micro_steps=2840 elapsed=3.6s lr=1.705200e-04 loss=4.7416 ppl=114.7103 acc=0.2650 tokens=8128.0000 +step=1440 micro_steps=2880 elapsed=3.6s lr=1.729200e-04 loss=4.7126 ppl=111.4812 acc=0.2663 tokens=8128.0000 +step=1460 micro_steps=2920 elapsed=3.6s lr=1.753200e-04 loss=4.7078 ppl=110.9196 acc=0.2674 tokens=8128.0000 +step=1480 micro_steps=2960 elapsed=3.6s lr=1.777200e-04 loss=4.6839 ppl=108.3107 acc=0.2680 tokens=8128.0000 +step=1500 micro_steps=3000 elapsed=3.6s lr=1.801200e-04 loss=4.6728 ppl=107.1491 acc=0.2698 tokens=8128.0000 +step=1520 micro_steps=3040 elapsed=3.6s lr=1.825200e-04 loss=4.6493 ppl=104.6331 acc=0.2710 tokens=8128.0000 +step=1540 micro_steps=3080 elapsed=3.6s lr=1.849200e-04 loss=4.6501 ppl=104.7323 acc=0.2707 tokens=8128.0000 +step=1560 micro_steps=3120 elapsed=3.6s lr=1.873200e-04 loss=4.6381 ppl=103.4611 acc=0.2718 tokens=8128.0000 +step=1580 micro_steps=3160 elapsed=3.6s lr=1.897200e-04 loss=4.6088 ppl=100.5290 acc=0.2751 tokens=8128.0000 +step=1600 micro_steps=3200 elapsed=3.6s lr=1.921200e-04 loss=4.6027 ppl=99.8427 acc=0.2747 tokens=8128.0000 +step=1620 micro_steps=3240 elapsed=3.6s lr=1.945200e-04 loss=4.5819 ppl=97.7759 acc=0.2764 tokens=8128.0000 +step=1640 micro_steps=3280 elapsed=3.6s lr=1.969200e-04 loss=4.5748 ppl=97.1006 acc=0.2762 tokens=8128.0000 +step=1660 micro_steps=3320 elapsed=3.6s lr=1.993200e-04 loss=4.5683 ppl=96.5347 acc=0.2770 tokens=8128.0000 +step=1680 micro_steps=3360 elapsed=3.6s lr=2.017200e-04 loss=4.5531 ppl=95.0791 acc=0.2777 tokens=8128.0000 +step=1700 micro_steps=3400 elapsed=3.6s lr=2.041200e-04 loss=4.5314 ppl=92.9927 acc=0.2800 tokens=8128.0000 +step=1720 micro_steps=3440 elapsed=3.6s lr=2.065200e-04 loss=4.5209 ppl=92.0741 acc=0.2796 tokens=8128.0000 +step=1740 micro_steps=3480 elapsed=3.6s lr=2.089200e-04 loss=4.5257 ppl=92.4572 acc=0.2809 tokens=8128.0000 +step=1760 micro_steps=3520 elapsed=3.6s lr=2.113200e-04 loss=4.5192 ppl=91.9197 acc=0.2803 tokens=8128.0000 +step=1780 micro_steps=3560 elapsed=3.6s lr=2.137200e-04 loss=4.4929 ppl=89.4739 acc=0.2837 tokens=8128.0000 +step=1800 micro_steps=3600 elapsed=3.6s lr=2.161200e-04 loss=4.4789 ppl=88.3597 acc=0.2846 tokens=8128.0000 +step=1820 micro_steps=3640 elapsed=3.6s lr=2.185200e-04 loss=4.4777 ppl=88.1819 acc=0.2846 tokens=8128.0000 +step=1840 micro_steps=3680 elapsed=3.6s lr=2.209200e-04 loss=4.4645 ppl=86.9802 acc=0.2859 tokens=8128.0000 +step=1860 micro_steps=3720 elapsed=3.6s lr=2.233200e-04 loss=4.4604 ppl=86.6318 acc=0.2861 tokens=8128.0000 +step=1880 micro_steps=3760 elapsed=3.6s lr=2.257200e-04 loss=4.4447 ppl=85.2599 acc=0.2866 tokens=8128.0000 +step=1900 micro_steps=3800 elapsed=3.6s lr=2.281200e-04 loss=4.4276 ppl=83.8217 acc=0.2876 tokens=8128.0000 +step=1920 micro_steps=3840 elapsed=3.6s lr=2.305200e-04 loss=4.4316 ppl=84.1305 acc=0.2883 tokens=8128.0000 +step=1940 micro_steps=3880 elapsed=3.6s lr=2.329200e-04 loss=4.4196 ppl=83.1516 acc=0.2876 tokens=8128.0000 +step=1960 micro_steps=3920 elapsed=3.6s lr=2.353200e-04 loss=4.4023 ppl=81.7102 acc=0.2905 tokens=8128.0000 +step=1980 micro_steps=3960 elapsed=3.6s lr=2.377200e-04 loss=4.4063 ppl=82.0973 acc=0.2905 tokens=8128.0000 +step=2000 micro_steps=4000 elapsed=3.6s lr=2.401200e-04 loss=4.3846 ppl=80.3016 acc=0.2917 tokens=8128.0000 +[sample step=2000] [CLS]t ), and at the moment - - you really don't know the same the best proportions for getting things attention this is going to get hurt. [SEP] mr justice debashev had a positive interest in most of the country. [SEP] it wants to be a useful change. [SEP] the principal names of veteran officers, groups of fans committees and members of congress, because they while not horse - goers will play an important role in obtaining substantial invoking over his federal support, there are no help for the united states to preserve iranian community capacity. [SEP] the pendulum of dieting out is underscore the size of gas [SEP] +step=2020 micro_steps=4040 elapsed=5.8s lr=2.425200e-04 loss=4.3824 ppl=80.1633 acc=0.2902 tokens=8128.0000 +step=2040 micro_steps=4080 elapsed=3.6s lr=2.449200e-04 loss=4.3680 ppl=78.9561 acc=0.2932 tokens=8128.0000 +step=2060 micro_steps=4120 elapsed=3.6s lr=2.473200e-04 loss=4.3571 ppl=78.1244 acc=0.2932 tokens=8128.0000 +step=2080 micro_steps=4160 elapsed=3.6s lr=2.497200e-04 loss=4.3752 ppl=79.5609 acc=0.2914 tokens=8128.0000 +step=2100 micro_steps=4200 elapsed=3.6s lr=2.521200e-04 loss=4.3523 ppl=77.7967 acc=0.2944 tokens=8128.0000 +step=2120 micro_steps=4240 elapsed=3.6s lr=2.545200e-04 loss=4.3403 ppl=76.8172 acc=0.2947 tokens=8128.0000 +step=2140 micro_steps=4280 elapsed=3.6s lr=2.569200e-04 loss=4.3307 ppl=76.0678 acc=0.2953 tokens=8128.0000 +step=2160 micro_steps=4320 elapsed=3.6s lr=2.593200e-04 loss=4.3392 ppl=76.7835 acc=0.2948 tokens=8128.0000 +step=2180 micro_steps=4360 elapsed=3.6s lr=2.617200e-04 loss=4.3338 ppl=76.3271 acc=0.2947 tokens=8128.0000 +step=2200 micro_steps=4400 elapsed=3.6s lr=2.641200e-04 loss=4.3230 ppl=75.4672 acc=0.2964 tokens=8128.0000 +step=2220 micro_steps=4440 elapsed=3.6s lr=2.665200e-04 loss=4.3048 ppl=74.1440 acc=0.2987 tokens=8128.0000 +step=2240 micro_steps=4480 elapsed=3.6s lr=2.689200e-04 loss=4.2885 ppl=72.9240 acc=0.2989 tokens=8128.0000 +step=2260 micro_steps=4520 elapsed=3.6s lr=2.713200e-04 loss=4.3005 ppl=73.8235 acc=0.2969 tokens=8128.0000 +step=2280 micro_steps=4560 elapsed=3.6s lr=2.737200e-04 loss=4.2877 ppl=72.8882 acc=0.2994 tokens=8128.0000 +step=2300 micro_steps=4600 elapsed=3.6s lr=2.761200e-04 loss=4.2833 ppl=72.5995 acc=0.2992 tokens=8128.0000 +step=2320 micro_steps=4640 elapsed=3.6s lr=2.785200e-04 loss=4.2780 ppl=72.1578 acc=0.2993 tokens=8128.0000 +step=2340 micro_steps=4680 elapsed=3.6s lr=2.809200e-04 loss=4.2667 ppl=71.3834 acc=0.2998 tokens=8128.0000 +step=2360 micro_steps=4720 elapsed=3.6s lr=2.833200e-04 loss=4.2594 ppl=70.8318 acc=0.3021 tokens=8128.0000 +step=2380 micro_steps=4760 elapsed=3.6s lr=2.857200e-04 loss=4.2395 ppl=69.4661 acc=0.3038 tokens=8128.0000 +step=2400 micro_steps=4800 elapsed=3.6s lr=2.881200e-04 loss=4.2583 ppl=70.7645 acc=0.3008 tokens=8128.0000 +step=2420 micro_steps=4840 elapsed=3.6s lr=2.905200e-04 loss=4.2262 ppl=68.5519 acc=0.3055 tokens=8128.0000 +step=2440 micro_steps=4880 elapsed=3.6s lr=2.929200e-04 loss=4.2411 ppl=69.5565 acc=0.3025 tokens=8128.0000 +step=2460 micro_steps=4920 elapsed=3.6s lr=2.953200e-04 loss=4.2159 ppl=67.8834 acc=0.3051 tokens=8128.0000 +step=2480 micro_steps=4960 elapsed=3.6s lr=2.977200e-04 loss=4.2350 ppl=69.1677 acc=0.3034 tokens=8128.0000 +step=2500 micro_steps=5000 elapsed=3.6s lr=3.000000e-04 loss=4.2098 ppl=67.4287 acc=0.3069 tokens=8128.0000 +step=2520 micro_steps=5040 elapsed=3.6s lr=3.000000e-04 loss=4.1991 ppl=66.6870 acc=0.3070 tokens=8128.0000 +step=2540 micro_steps=5080 elapsed=3.6s lr=3.000000e-04 loss=4.2085 ppl=67.3566 acc=0.3055 tokens=8128.0000 +step=2560 micro_steps=5120 elapsed=3.6s lr=3.000000e-04 loss=4.1952 ppl=66.4172 acc=0.3073 tokens=8128.0000 +step=2580 micro_steps=5160 elapsed=3.6s lr=3.000000e-04 loss=4.1903 ppl=66.1340 acc=0.3071 tokens=8128.0000 +step=2600 micro_steps=5200 elapsed=3.6s lr=3.000000e-04 loss=4.1832 ppl=65.6817 acc=0.3084 tokens=8128.0000 +step=2620 micro_steps=5240 elapsed=3.6s lr=3.000000e-04 loss=4.1843 ppl=65.7183 acc=0.3079 tokens=8128.0000 +step=2640 micro_steps=5280 elapsed=3.6s lr=3.000000e-04 loss=4.1928 ppl=66.2759 acc=0.3070 tokens=8128.0000 +step=2660 micro_steps=5320 elapsed=3.6s lr=3.000000e-04 loss=4.1723 ppl=64.9454 acc=0.3093 tokens=8128.0000 +step=2680 micro_steps=5360 elapsed=3.6s lr=3.000000e-04 loss=4.1549 ppl=63.8198 acc=0.3101 tokens=8128.0000 +step=2700 micro_steps=5400 elapsed=3.6s lr=3.000000e-04 loss=4.1537 ppl=63.7657 acc=0.3095 tokens=8128.0000 +step=2720 micro_steps=5440 elapsed=3.6s lr=3.000000e-04 loss=4.1471 ppl=63.3684 acc=0.3103 tokens=8128.0000 +step=2740 micro_steps=5480 elapsed=3.6s lr=3.000000e-04 loss=4.1499 ppl=63.5364 acc=0.3112 tokens=8128.0000 +step=2760 micro_steps=5520 elapsed=3.6s lr=3.000000e-04 loss=4.1403 ppl=62.9134 acc=0.3120 tokens=8128.0000 +step=2780 micro_steps=5560 elapsed=3.6s lr=3.000000e-04 loss=4.1390 ppl=62.8143 acc=0.3123 tokens=8128.0000 +step=2800 micro_steps=5600 elapsed=3.6s lr=3.000000e-04 loss=4.1324 ppl=62.4175 acc=0.3124 tokens=8128.0000 +step=2820 micro_steps=5640 elapsed=3.6s lr=3.000000e-04 loss=4.1207 ppl=61.6718 acc=0.3127 tokens=8128.0000 +step=2840 micro_steps=5680 elapsed=3.6s lr=3.000000e-04 loss=4.1225 ppl=61.7597 acc=0.3127 tokens=8128.0000 +step=2860 micro_steps=5720 elapsed=3.6s lr=3.000000e-04 loss=4.1187 ppl=61.5916 acc=0.3140 tokens=8128.0000 +step=2880 micro_steps=5760 elapsed=3.6s lr=3.000000e-04 loss=4.1156 ppl=61.3453 acc=0.3132 tokens=8128.0000 +step=2900 micro_steps=5800 elapsed=3.6s lr=3.000000e-04 loss=4.0986 ppl=60.3072 acc=0.3161 tokens=8128.0000 +step=2920 micro_steps=5840 elapsed=3.6s lr=3.000000e-04 loss=4.1038 ppl=60.6632 acc=0.3147 tokens=8128.0000 +step=2940 micro_steps=5880 elapsed=3.6s lr=3.000000e-04 loss=4.1071 ppl=60.8467 acc=0.3137 tokens=8128.0000 +step=2960 micro_steps=5920 elapsed=3.6s lr=3.000000e-04 loss=4.0904 ppl=59.8346 acc=0.3163 tokens=8128.0000 +step=2980 micro_steps=5960 elapsed=3.6s lr=3.000000e-04 loss=4.1033 ppl=60.6203 acc=0.3137 tokens=8128.0000 +step=3000 micro_steps=6000 elapsed=3.6s lr=3.000000e-04 loss=4.0854 ppl=59.5390 acc=0.3165 tokens=8128.0000 +[sample step=3000] [CLS] hapless crude, trading from $38. 26 a barrel on reports on immeas roderick lee - - who had seen the big 12 experience when people arrived to check the stock. [SEP] attempted asylum ringleader s. b. son - in - law also linked col. umar with gen. rauci radan, who has until september 11 to attend the celebre jury hijacker in mount wootton, north yorkshire, and recently flown to new york, cuba with an apology from 18 self - proclaimed owners of a home where 13 - year - old gareth culbert abused his job as one of the first [SEP] +step=3020 micro_steps=6040 elapsed=5.8s lr=3.000000e-04 loss=4.0828 ppl=59.3799 acc=0.3161 tokens=8128.0000 +step=3040 micro_steps=6080 elapsed=3.6s lr=3.000000e-04 loss=4.0786 ppl=59.1482 acc=0.3171 tokens=8128.0000 +step=3060 micro_steps=6120 elapsed=3.6s lr=3.000000e-04 loss=4.0716 ppl=58.7107 acc=0.3180 tokens=8128.0000 +step=3080 micro_steps=6160 elapsed=3.6s lr=3.000000e-04 loss=4.0767 ppl=59.0115 acc=0.3177 tokens=8128.0000 +step=3100 micro_steps=6200 elapsed=3.6s lr=3.000000e-04 loss=4.0632 ppl=58.2278 acc=0.3177 tokens=8128.0000 +step=3120 micro_steps=6240 elapsed=3.6s lr=3.000000e-04 loss=4.0738 ppl=58.8580 acc=0.3168 tokens=8128.0000 +step=3140 micro_steps=6280 elapsed=3.6s lr=3.000000e-04 loss=4.0672 ppl=58.4934 acc=0.3178 tokens=8128.0000 +step=3160 micro_steps=6320 elapsed=3.6s lr=3.000000e-04 loss=4.0560 ppl=57.8210 acc=0.3183 tokens=8128.0000 +step=3180 micro_steps=6360 elapsed=3.6s lr=3.000000e-04 loss=4.0523 ppl=57.6100 acc=0.3202 tokens=8128.0000 +step=3200 micro_steps=6400 elapsed=3.6s lr=3.000000e-04 loss=4.0588 ppl=57.9823 acc=0.3185 tokens=8128.0000 +step=3220 micro_steps=6440 elapsed=3.6s lr=3.000000e-04 loss=4.0476 ppl=57.3425 acc=0.3196 tokens=8128.0000 +step=3240 micro_steps=6480 elapsed=3.6s lr=3.000000e-04 loss=4.0504 ppl=57.5045 acc=0.3200 tokens=8128.0000 +step=3260 micro_steps=6520 elapsed=3.6s lr=3.000000e-04 loss=4.0352 ppl=56.6460 acc=0.3207 tokens=8128.0000 +step=3280 micro_steps=6560 elapsed=3.6s lr=3.000000e-04 loss=4.0359 ppl=56.6621 acc=0.3211 tokens=8128.0000 +step=3300 micro_steps=6600 elapsed=3.6s lr=3.000000e-04 loss=4.0452 ppl=57.1848 acc=0.3194 tokens=8128.0000 +step=3320 micro_steps=6640 elapsed=3.6s lr=3.000000e-04 loss=4.0308 ppl=56.3376 acc=0.3218 tokens=8128.0000 +step=3340 micro_steps=6680 elapsed=3.6s lr=3.000000e-04 loss=4.0391 ppl=56.8609 acc=0.3209 tokens=8128.0000 +step=3360 micro_steps=6720 elapsed=3.6s lr=3.000000e-04 loss=4.0293 ppl=56.2732 acc=0.3222 tokens=8128.0000 +step=3380 micro_steps=6760 elapsed=3.6s lr=3.000000e-04 loss=4.0174 ppl=55.5969 acc=0.3221 tokens=8128.0000 +step=3400 micro_steps=6800 elapsed=3.6s lr=3.000000e-04 loss=4.0322 ppl=56.4517 acc=0.3218 tokens=8128.0000 +step=3420 micro_steps=6840 elapsed=3.6s lr=3.000000e-04 loss=4.0103 ppl=55.2042 acc=0.3222 tokens=8128.0000 +step=3440 micro_steps=6880 elapsed=3.6s lr=3.000000e-04 loss=4.0159 ppl=55.5478 acc=0.3234 tokens=8128.0000 +step=3460 micro_steps=6920 elapsed=3.6s lr=3.000000e-04 loss=4.0053 ppl=54.9606 acc=0.3242 tokens=8128.0000 +step=3480 micro_steps=6960 elapsed=3.6s lr=3.000000e-04 loss=4.0200 ppl=55.7635 acc=0.3221 tokens=8128.0000 +step=3500 micro_steps=7000 elapsed=3.6s lr=3.000000e-04 loss=3.9826 ppl=53.7106 acc=0.3250 tokens=8128.0000 +step=3520 micro_steps=7040 elapsed=3.6s lr=3.000000e-04 loss=4.0046 ppl=54.9312 acc=0.3248 tokens=8128.0000 +step=3540 micro_steps=7080 elapsed=3.6s lr=3.000000e-04 loss=3.9995 ppl=54.6437 acc=0.3252 tokens=8128.0000 +step=3560 micro_steps=7120 elapsed=3.6s lr=3.000000e-04 loss=3.9930 ppl=54.2816 acc=0.3244 tokens=8128.0000 +step=3580 micro_steps=7160 elapsed=3.6s lr=3.000000e-04 loss=3.9871 ppl=53.9546 acc=0.3248 tokens=8128.0000 +step=3600 micro_steps=7200 elapsed=3.6s lr=3.000000e-04 loss=3.9826 ppl=53.7258 acc=0.3260 tokens=8128.0000 +step=3620 micro_steps=7240 elapsed=3.6s lr=3.000000e-04 loss=3.9888 ppl=54.0497 acc=0.3256 tokens=8128.0000 +step=3640 micro_steps=7280 elapsed=3.6s lr=3.000000e-04 loss=3.9819 ppl=53.6996 acc=0.3260 tokens=8128.0000 +step=3660 micro_steps=7320 elapsed=3.6s lr=3.000000e-04 loss=3.9797 ppl=53.5662 acc=0.3270 tokens=8128.0000 +step=3680 micro_steps=7360 elapsed=3.6s lr=3.000000e-04 loss=3.9860 ppl=53.8878 acc=0.3255 tokens=8128.0000 +step=3700 micro_steps=7400 elapsed=3.6s lr=3.000000e-04 loss=3.9782 ppl=53.4761 acc=0.3250 tokens=8128.0000 +step=3720 micro_steps=7440 elapsed=3.6s lr=3.000000e-04 loss=3.9659 ppl=52.8435 acc=0.3281 tokens=8128.0000 +step=3740 micro_steps=7480 elapsed=3.6s lr=3.000000e-04 loss=3.9718 ppl=53.1304 acc=0.3275 tokens=8128.0000 +step=3760 micro_steps=7520 elapsed=3.6s lr=3.000000e-04 loss=3.9885 ppl=54.0326 acc=0.3251 tokens=8128.0000 +step=3780 micro_steps=7560 elapsed=3.6s lr=3.000000e-04 loss=3.9540 ppl=52.2093 acc=0.3283 tokens=8128.0000 +step=3800 micro_steps=7600 elapsed=3.6s lr=3.000000e-04 loss=3.9595 ppl=52.4764 acc=0.3275 tokens=8128.0000 +step=3820 micro_steps=7640 elapsed=3.6s lr=3.000000e-04 loss=3.9700 ppl=53.0521 acc=0.3263 tokens=8128.0000 +step=3840 micro_steps=7680 elapsed=3.6s lr=3.000000e-04 loss=3.9764 ppl=53.4071 acc=0.3256 tokens=8128.0000 +step=3860 micro_steps=7720 elapsed=3.6s lr=3.000000e-04 loss=3.9489 ppl=51.9420 acc=0.3287 tokens=8128.0000 +step=3880 micro_steps=7760 elapsed=3.6s lr=3.000000e-04 loss=3.9492 ppl=51.9533 acc=0.3290 tokens=8128.0000 +step=3900 micro_steps=7800 elapsed=3.6s lr=3.000000e-04 loss=3.9630 ppl=52.6557 acc=0.3273 tokens=8128.0000 +step=3920 micro_steps=7840 elapsed=3.6s lr=3.000000e-04 loss=3.9416 ppl=51.5544 acc=0.3302 tokens=8128.0000 +step=3940 micro_steps=7880 elapsed=3.6s lr=3.000000e-04 loss=3.9534 ppl=52.1683 acc=0.3286 tokens=8128.0000 +step=3960 micro_steps=7920 elapsed=3.6s lr=3.000000e-04 loss=3.9490 ppl=51.9456 acc=0.3284 tokens=8128.0000 +step=3980 micro_steps=7960 elapsed=3.6s lr=3.000000e-04 loss=3.9503 ppl=52.0025 acc=0.3275 tokens=8128.0000 +step=4000 micro_steps=8000 elapsed=3.6s lr=3.000000e-04 loss=3.9315 ppl=51.0298 acc=0.3307 tokens=8128.0000 +[sample step=4000] [CLS] such a showdown, which ended some years later with a trip to the united states. [SEP] worsening the pain in americans - - and hopefully inapturating it - - is hard on those people. [SEP] at one point, it would rise near the corset airport in post - communist north africa, drawing viewers who struggle to cope with tourist hikes. [SEP] he had sheltered some $40, 000 for a farmer from working his way to a nearby park instead of labeling a panda or watching the wild boar (30 feet) instead of the man. [SEP] is affluensky's demand definitely not worth? [SEP] [SEP] +step=4020 micro_steps=8040 elapsed=5.8s lr=3.000000e-04 loss=3.9312 ppl=51.0517 acc=0.3308 tokens=8128.0000 +step=4040 micro_steps=8080 elapsed=3.6s lr=3.000000e-04 loss=3.9361 ppl=51.2798 acc=0.3303 tokens=8128.0000 +step=4060 micro_steps=8120 elapsed=3.6s lr=3.000000e-04 loss=3.9501 ppl=52.0138 acc=0.3285 tokens=8128.0000 +step=4080 micro_steps=8160 elapsed=3.6s lr=3.000000e-04 loss=3.9394 ppl=51.4709 acc=0.3286 tokens=8128.0000 +step=4100 micro_steps=8200 elapsed=3.6s lr=3.000000e-04 loss=3.9253 ppl=50.7103 acc=0.3307 tokens=8128.0000 +step=4120 micro_steps=8240 elapsed=3.6s lr=3.000000e-04 loss=3.9292 ppl=50.9211 acc=0.3316 tokens=8128.0000 +step=4140 micro_steps=8280 elapsed=3.6s lr=3.000000e-04 loss=3.9309 ppl=51.0085 acc=0.3309 tokens=8128.0000 +step=4160 micro_steps=8320 elapsed=3.6s lr=3.000000e-04 loss=3.9245 ppl=50.7222 acc=0.3310 tokens=8128.0000 +step=4180 micro_steps=8360 elapsed=3.6s lr=3.000000e-04 loss=3.9219 ppl=50.5655 acc=0.3307 tokens=8128.0000 +step=4200 micro_steps=8400 elapsed=3.6s lr=3.000000e-04 loss=3.9287 ppl=50.9071 acc=0.3310 tokens=8128.0000 +step=4220 micro_steps=8440 elapsed=3.6s lr=3.000000e-04 loss=3.9173 ppl=50.3085 acc=0.3305 tokens=8128.0000 +step=4240 micro_steps=8480 elapsed=3.6s lr=3.000000e-04 loss=3.9112 ppl=50.0233 acc=0.3331 tokens=8128.0000 +step=4260 micro_steps=8520 elapsed=3.6s lr=3.000000e-04 loss=3.9256 ppl=50.7239 acc=0.3311 tokens=8128.0000 +step=4280 micro_steps=8560 elapsed=3.6s lr=3.000000e-04 loss=3.9058 ppl=49.7594 acc=0.3336 tokens=8128.0000 +step=4300 micro_steps=8600 elapsed=3.6s lr=3.000000e-04 loss=3.8825 ppl=48.6190 acc=0.3352 tokens=8128.0000 +step=4320 micro_steps=8640 elapsed=3.6s lr=3.000000e-04 loss=3.9073 ppl=49.8234 acc=0.3320 tokens=8128.0000 +step=4340 micro_steps=8680 elapsed=3.6s lr=3.000000e-04 loss=3.9016 ppl=49.5402 acc=0.3324 tokens=8128.0000 +step=4360 micro_steps=8720 elapsed=3.6s lr=3.000000e-04 loss=3.9019 ppl=49.5451 acc=0.3337 tokens=8128.0000 +step=4380 micro_steps=8760 elapsed=3.6s lr=3.000000e-04 loss=3.8976 ppl=49.3407 acc=0.3327 tokens=8128.0000 +step=4400 micro_steps=8800 elapsed=3.6s lr=3.000000e-04 loss=3.8919 ppl=49.0610 acc=0.3336 tokens=8128.0000 +step=4420 micro_steps=8840 elapsed=3.6s lr=3.000000e-04 loss=3.8915 ppl=49.0252 acc=0.3344 tokens=8128.0000 +step=4440 micro_steps=8880 elapsed=3.6s lr=3.000000e-04 loss=3.8883 ppl=48.8853 acc=0.3333 tokens=8128.0000 +step=4460 micro_steps=8920 elapsed=3.6s lr=3.000000e-04 loss=3.9000 ppl=49.4839 acc=0.3336 tokens=8128.0000 +step=4480 micro_steps=8960 elapsed=3.6s lr=3.000000e-04 loss=3.8926 ppl=49.0913 acc=0.3341 tokens=8128.0000 +step=4500 micro_steps=9000 elapsed=3.6s lr=3.000000e-04 loss=3.8436 ppl=47.3114 acc=0.3418 tokens=8128.0000 +step=4520 micro_steps=9040 elapsed=3.6s lr=3.000000e-04 loss=3.8916 ppl=49.0472 acc=0.3343 tokens=8128.0000 +step=4540 micro_steps=9080 elapsed=3.6s lr=3.000000e-04 loss=3.8798 ppl=48.4476 acc=0.3358 tokens=8128.0000 +step=4560 micro_steps=9120 elapsed=3.6s lr=3.000000e-04 loss=3.8811 ppl=48.5576 acc=0.3355 tokens=8128.0000 +step=4580 micro_steps=9160 elapsed=3.6s lr=3.000000e-04 loss=3.8873 ppl=48.8260 acc=0.3343 tokens=8128.0000 +step=4600 micro_steps=9200 elapsed=3.6s lr=3.000000e-04 loss=3.8902 ppl=48.9647 acc=0.3335 tokens=8128.0000 +step=4620 micro_steps=9240 elapsed=3.6s lr=3.000000e-04 loss=3.8757 ppl=48.2724 acc=0.3353 tokens=8128.0000 +step=4640 micro_steps=9280 elapsed=3.6s lr=3.000000e-04 loss=3.8801 ppl=48.5090 acc=0.3355 tokens=8128.0000 +step=4660 micro_steps=9320 elapsed=3.6s lr=3.000000e-04 loss=3.8660 ppl=47.8119 acc=0.3357 tokens=8128.0000 +step=4680 micro_steps=9360 elapsed=3.6s lr=3.000000e-04 loss=3.8870 ppl=48.8011 acc=0.3352 tokens=8128.0000 +step=4700 micro_steps=9400 elapsed=3.6s lr=3.000000e-04 loss=3.8677 ppl=47.9033 acc=0.3370 tokens=8128.0000 +step=4720 micro_steps=9440 elapsed=3.6s lr=3.000000e-04 loss=3.8797 ppl=48.4573 acc=0.3350 tokens=8128.0000 +step=4740 micro_steps=9480 elapsed=3.6s lr=3.000000e-04 loss=3.8766 ppl=48.3031 acc=0.3353 tokens=8128.0000 +step=4760 micro_steps=9520 elapsed=3.6s lr=3.000000e-04 loss=3.8714 ppl=48.0545 acc=0.3365 tokens=8128.0000 +step=4780 micro_steps=9560 elapsed=3.6s lr=3.000000e-04 loss=3.8526 ppl=47.1621 acc=0.3381 tokens=8128.0000 +step=4800 micro_steps=9600 elapsed=3.6s lr=3.000000e-04 loss=3.8639 ppl=47.6841 acc=0.3374 tokens=8128.0000 +step=4820 micro_steps=9640 elapsed=3.6s lr=3.000000e-04 loss=3.8566 ppl=47.3693 acc=0.3369 tokens=8128.0000 +step=4840 micro_steps=9680 elapsed=3.6s lr=3.000000e-04 loss=3.8527 ppl=47.1919 acc=0.3392 tokens=8128.0000 +step=4860 micro_steps=9720 elapsed=3.6s lr=3.000000e-04 loss=3.8581 ppl=47.4503 acc=0.3370 tokens=8128.0000 +step=4880 micro_steps=9760 elapsed=3.6s lr=3.000000e-04 loss=3.8667 ppl=47.8499 acc=0.3356 tokens=8128.0000 +step=4900 micro_steps=9800 elapsed=3.6s lr=3.000000e-04 loss=3.8576 ppl=47.4053 acc=0.3376 tokens=8128.0000 +step=4920 micro_steps=9840 elapsed=3.6s lr=3.000000e-04 loss=3.8391 ppl=46.5728 acc=0.3389 tokens=8128.0000 +step=4940 micro_steps=9880 elapsed=3.6s lr=3.000000e-04 loss=3.8671 ppl=47.8644 acc=0.3354 tokens=8128.0000 +step=4960 micro_steps=9920 elapsed=3.6s lr=3.000000e-04 loss=3.8590 ppl=47.4656 acc=0.3372 tokens=8128.0000 +step=4980 micro_steps=9960 elapsed=3.6s lr=3.000000e-04 loss=3.8433 ppl=46.7323 acc=0.3388 tokens=8128.0000 +step=5000 micro_steps=10000 elapsed=3.6s lr=3.000000e-04 loss=3.8529 ppl=47.1858 acc=0.3383 tokens=8128.0000 +[sample step=5000] [CLS] siddiqui, said he felt that mr. haidl's decision had been influenced at best by popular experiences, his long - time opponent in power and friendship among iraqis, his family and friends. [SEP] the oft found the us consumer credit agency in a weak showing in the early 1990s as well as a resurgence by the htc of the u. s. - based marketing firm roche. [SEP] tidal energy's developers will also have to acquire time warner network for up to $1 billion. [SEP] 0428: will clorox and epert 8 sell it in jars? [SEP] until now, ebay ' [SEP] +step=5020 micro_steps=10040 elapsed=5.8s lr=3.000000e-04 loss=3.8548 ppl=47.2749 acc=0.3385 tokens=8128.0000 +step=5040 micro_steps=10080 elapsed=3.6s lr=3.000000e-04 loss=3.8402 ppl=46.5920 acc=0.3385 tokens=8128.0000 +step=5060 micro_steps=10120 elapsed=3.6s lr=3.000000e-04 loss=3.8436 ppl=46.7635 acc=0.3386 tokens=8128.0000 +step=5080 micro_steps=10160 elapsed=3.6s lr=3.000000e-04 loss=3.8557 ppl=47.3276 acc=0.3375 tokens=8128.0000 +step=5100 micro_steps=10200 elapsed=3.6s lr=3.000000e-04 loss=3.8403 ppl=46.5893 acc=0.3387 tokens=8128.0000 +step=5120 micro_steps=10240 elapsed=3.6s lr=3.000000e-04 loss=3.8308 ppl=46.1426 acc=0.3396 tokens=8128.0000 +step=5140 micro_steps=10280 elapsed=3.6s lr=3.000000e-04 loss=3.8295 ppl=46.0875 acc=0.3423 tokens=8128.0000 +step=5160 micro_steps=10320 elapsed=3.6s lr=3.000000e-04 loss=3.8385 ppl=46.4952 acc=0.3402 tokens=8128.0000 +step=5180 micro_steps=10360 elapsed=3.6s lr=3.000000e-04 loss=3.8380 ppl=46.4819 acc=0.3401 tokens=8128.0000 +step=5200 micro_steps=10400 elapsed=3.6s lr=3.000000e-04 loss=3.8301 ppl=46.1601 acc=0.3395 tokens=8128.0000 +step=5220 micro_steps=10440 elapsed=3.6s lr=3.000000e-04 loss=3.8380 ppl=46.5034 acc=0.3395 tokens=8128.0000 +step=5240 micro_steps=10480 elapsed=3.6s lr=3.000000e-04 loss=3.8326 ppl=46.2529 acc=0.3408 tokens=8128.0000 +step=5260 micro_steps=10520 elapsed=3.6s lr=3.000000e-04 loss=3.8225 ppl=45.7632 acc=0.3410 tokens=8128.0000 +step=5280 micro_steps=10560 elapsed=3.6s lr=3.000000e-04 loss=3.8151 ppl=45.4480 acc=0.3414 tokens=8128.0000 +step=5300 micro_steps=10600 elapsed=3.6s lr=3.000000e-04 loss=3.8284 ppl=46.0495 acc=0.3399 tokens=8128.0000 +step=5320 micro_steps=10640 elapsed=3.6s lr=3.000000e-04 loss=3.8339 ppl=46.2933 acc=0.3391 tokens=8128.0000 +step=5340 micro_steps=10680 elapsed=3.6s lr=3.000000e-04 loss=3.8308 ppl=46.1532 acc=0.3395 tokens=8128.0000 +step=5360 micro_steps=10720 elapsed=3.6s lr=3.000000e-04 loss=3.8233 ppl=45.8144 acc=0.3416 tokens=8128.0000 +step=5380 micro_steps=10760 elapsed=3.6s lr=3.000000e-04 loss=3.8281 ppl=46.0230 acc=0.3399 tokens=8128.0000 +step=5400 micro_steps=10800 elapsed=3.6s lr=3.000000e-04 loss=3.8238 ppl=45.8492 acc=0.3411 tokens=8128.0000 +step=5420 micro_steps=10840 elapsed=3.6s lr=3.000000e-04 loss=3.8261 ppl=45.9348 acc=0.3401 tokens=8128.0000 +step=5440 micro_steps=10880 elapsed=3.6s lr=3.000000e-04 loss=3.8114 ppl=45.2710 acc=0.3412 tokens=8128.0000 +step=5460 micro_steps=10920 elapsed=3.6s lr=3.000000e-04 loss=3.8170 ppl=45.5219 acc=0.3415 tokens=8128.0000 +step=5480 micro_steps=10960 elapsed=3.6s lr=3.000000e-04 loss=3.8110 ppl=45.2390 acc=0.3422 tokens=8128.0000 diff --git a/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len256_gbs512_4gpu_10k_save1k_20260523.train.pid b/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len256_gbs512_4gpu_10k_save1k_20260523.train.pid new file mode 100644 index 0000000000000000000000000000000000000000..4e1f6fec7d2062fd6677a3712d2656970d143b16 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len256_gbs512_4gpu_10k_save1k_20260523.train.pid @@ -0,0 +1 @@ +994417 diff --git a/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_shufchunks_len128_gbs512_8gpu_1m.log b/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_shufchunks_len128_gbs512_8gpu_1m.log new file mode 100644 index 0000000000000000000000000000000000000000..245c773ec4a30879b4d077ed5db8d24939c64d7b --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_shufchunks_len128_gbs512_8gpu_1m.log @@ -0,0 +1,2976 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank2]:[W509 22:02:24.903514400 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:02:25.992100756 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:02:25.021424184 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:02:25.032195161 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:02:25.124739613 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:02:25.125023982 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:02:25.125330112 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:12:25.024046692 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49834, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:12:25.024124656 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49834, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8526f2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f8576e4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f857b4eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f857b4ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f857b48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f857b48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f857b48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8527c6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8527c6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f8527c73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8527c744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8527c8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f857b47c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f857b48bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f857ab569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f857b49a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f857b49b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f85836a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f8582db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f858f6f91ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f858f6f928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:12:25.091028791 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49824, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:12:25.091099803 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49824, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f70b136c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f7101333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f71059cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f71059d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f710597392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f710597392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f710597392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f70b214717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f70b21532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f70b2158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f70b21594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f70b216913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f710596165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f7105970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f710503b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f710597f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f710598076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f710db8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f710d29911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f7119b8a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f7119b8a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:12:25.102534042 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49870, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:12:25.102602119 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49870, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fc1f8e14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7fc248d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fc24d3cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fc24d3d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc24d37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc24d37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc24d37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fc1f9b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fc1f9b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7fc1f9b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fc1f9b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fc1f9b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7fc24d36165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7fc24d370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7fc24ca3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7fc24d37f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7fc24d38076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7fc25558975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7fc254c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7fc2615dc1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7fc2615dc28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 22:12:25.136819558 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49832, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:12:25.136886184 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49832, remote=[localhost]:29672) timed out after 600000ms +[rank4]:[W509 22:12:25.139385972 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49882, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:12:25.139450726 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49882, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fe94e16c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fe99e133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fe9a27cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fe9a27d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe9a277392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe9a277392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe9a277392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fe94ef4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fe94ef532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fe94ef58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fe94ef594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fe94ef6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fe9a276165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fe9a2770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fe9a1e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fe9a277f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fe9a278076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fe9aa98975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fe9aa09911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fe9b68fd1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fe9b68fd28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fa31f014818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fa36ef33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fa3735cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fa3735d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa37357392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa37357392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa37357392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fa31fd4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fa31fd532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fa31fd58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fa31fd594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fa31fd6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fa37356165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fa373570cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fa372c3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fa37357f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fa37358076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fa37b78975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fa37ae9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fa3877ec1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fa3877ec28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:12:25.195347105 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49838, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:12:25.195419927 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49838, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff114f2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7ff164e4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff1694eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff1694ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff16948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff16948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff16948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff115c6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff115c6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7ff115c73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff115c744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff115c8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7ff16947c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7ff16948bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7ff168b569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7ff16949a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7ff16949b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7ff1716a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7ff170db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7ff17d6751ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7ff17d67528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:12:25.229821133 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:49868, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:12:25.229890555 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:49868, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f645512f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f64a504ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f64a96eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f64a96ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f64a968e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f64a968e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f64a968e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f6455e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f6455e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f6455e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f6455e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f6455e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f64a967c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f64a968bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f64a8d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f64a969a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f64a969b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f64b18a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f64b0fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f64bd8ca1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f64bd8ca28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:12:25.576000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10301 closing signal SIGTERM +W0509 22:12:25.577000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10302 closing signal SIGTERM +W0509 22:12:25.577000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10303 closing signal SIGTERM +W0509 22:12:25.577000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10305 closing signal SIGTERM +W0509 22:12:25.578000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10306 closing signal SIGTERM +W0509 22:12:25.578000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10307 closing signal SIGTERM +W0509 22:12:25.578000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10308 closing signal SIGTERM +E0509 22:12:25.857000 10234 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10304) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:12:25 + host : t-20260510060145-nqj8p-worker-0.t-20260510060145-nqj8p-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10304) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank3]:[W509 22:14:14.468265026 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:14:14.492980825 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:14:14.617032621 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:14:14.650907960 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:14:14.652302722 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:14:14.671529777 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:14:14.677239340 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:24:14.586269790 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53798, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:24:14.586353439 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53798, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f2f85487818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f2fd544ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f2fd9aeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f2fd9aebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2fd9a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2fd9a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2fd9a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f2f8626217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f2f8626e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f2f86273cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f2f862744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f2f8628413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f2fd9a7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f2fd9a8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f2fd91569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f2fd9a9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f2fd9a9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f2fe1ca475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f2fe13b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f2fedc201ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f2fedc2028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 22:24:14.615832435 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53850, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:24:14.615896844 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53850, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8aa9614818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f8af9533c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8afdbcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8afdbd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8afdb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8afdb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8afdb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8aaa34717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8aaa3532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f8aaa358cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8aaa3594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8aaa36913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f8afdb6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f8afdb70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f8afd23b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f8afdb7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f8afdb8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f8b05d8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f8b0549911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f8b11e021ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f8b11e0228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:24:14.644143850 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53794, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:24:14.644208287 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53794, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f7fe5b6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f8035b33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f803a1cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f803a1d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f803a17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f803a17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f803a17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f7fe694717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f7fe69532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f7fe6958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f7fe69594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f7fe696913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f803a16165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f803a170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f803983b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f803a17f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f803a18076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f804238975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f8041a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f804e3281ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f804e32828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:24:14.667904913 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53826, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:24:14.667967605 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53826, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f80bd92f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f810d84ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8111eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8111eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8111e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8111e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8111e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f80be66217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f80be66e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f80be673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f80be6744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f80be68413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f8111e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f8111e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f81115569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f8111e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f8111e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f811a0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f81197b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f81260c41ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f81260c428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:24:14.699614623 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53862, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:24:14.699682479 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53862, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f9b6596c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f9bb5933c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f9bb9fcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f9bb9fd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9bb9f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9bb9f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9bb9f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f9b6674717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f9b667532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f9b66758cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f9b667594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f9b6676913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f9bb9f6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f9bb9f70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f9bb963b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f9bb9f7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f9bb9f8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f9bc218975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f9bc189911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f9bce1311ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f9bce13128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 22:24:14.756407211 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53796, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:24:14.756481862 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53796, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f0477b2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f04c7a4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f04cc0eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f04cc0ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f04cc08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f04cc08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f04cc08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f047886217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f047886e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f0478873cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f04788744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f047888413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f04cc07c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f04cc08bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f04cb7569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f04cc09a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f04cc09b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f04d42a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f04d39b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f04e02951ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f04e029528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:24:14.782580384 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:53842, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:24:14.782679474 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:53842, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fafc8c87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7fb018c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb01d2eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb01d2ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb01d28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb01d28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb01d28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fafc9a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fafc9a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7fafc9a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fafc9a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fafc9a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7fb01d27c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7fb01d28bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7fb01c9569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7fb01d29a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7fb01d29b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7fb0254a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7fb024bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7fb0314191ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7fb03141928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:24:15.019000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:24:15.020000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 22:24:15.020000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 22:24:15.021000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 22:24:15.021000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0509 22:24:15.264000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10352) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:24:15 + host : t-20260510061242-l4fbq-worker-0.t-20260510061242-l4fbq-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_22:24:15 + host : t-20260510061242-l4fbq-worker-0.t-20260510061242-l4fbq-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:24:15 + host : t-20260510061242-l4fbq-worker-0.t-20260510061242-l4fbq-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10352) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank5]:[W509 22:30:16.232964523 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:30:16.463927747 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:30:16.520656796 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:30:16.548340197 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:30:16.548766929 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:30:16.548788346 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:30:16.570534599 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:40:16.282615433 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54044, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:40:16.282702672 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54044, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f370fe14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f375fd33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f37643cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f37643d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f376437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f376437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f376437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3710b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3710b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f3710b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3710b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3710b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f376436165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f3764370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f3763a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f376437f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f376438076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f376c58975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f376bc9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f37785fb1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f37785fb28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:40:16.502034729 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54076, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:40:16.502110730 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54076, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f62bac87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f630ac4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f630f2eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f630f2ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f630f28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f630f28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f630f28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f62bba6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f62bba6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f62bba73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f62bba744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f62bba8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f630f27c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f630f28bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f630e9569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f630f29a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f630f29b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f63174a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f6316bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f63234241ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f632342428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:40:16.542613647 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54042, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:40:16.542686946 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54042, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f85b2487818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f860244ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8606aeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8606aebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8606a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8606a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8606a8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f85b326217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f85b326e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f85b3273cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f85b32744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f85b328413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f8606a7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f8606a8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f86061569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f8606a9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f8606a9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f860eca475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f860e3b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f861ac231ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f861ac2328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:40:16.582043353 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54046, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:40:16.582108380 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54046, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fd007414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7fd057333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fd05b9cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fd05b9d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd05b97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd05b97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd05b97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fd00814717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fd0081532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7fd008158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fd0081594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fd00816913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7fd05b96165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7fd05b970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7fd05b03b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7fd05b97f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7fd05b98076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7fd063b8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7fd06329911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7fd06fc081ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7fd06fc0828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:40:16.605028029 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54096, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:40:16.605089752 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54096, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3364e14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f33b4d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f33b93cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f33b93d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33b937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33b937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33b937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3365b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3365b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f3365b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3365b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3365b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f33b936165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f33b9370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f33b8a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f33b937f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f33b938076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f33c158975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f33c0c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f33cd5be1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f33cd5be28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 22:40:16.652055287 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54084, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:40:16.652128514 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54084, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 22:40:16.653039660 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:54082, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:40:16.653101862 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:54082, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f493596c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f4985933c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f4989fcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f4989fd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4989f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4989f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4989f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f493674717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f49367532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f4936758cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f49367594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f493676913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f4989f6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f4989f70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f498963b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f4989f7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f4989f8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f499218975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f499189911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f499e0ff1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f499e0ff28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f145912f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f14a904ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f14ad6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f14ad6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f14ad68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f14ad68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f14ad68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f1459e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f1459e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f1459e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f1459e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f1459e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f14ad67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f14ad68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f14acd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f14ad69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f14ad69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f14b58a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f14b4fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f14c18b41ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f14c18b428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:40:16.814000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:40:16.815000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 22:40:16.816000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 22:40:16.816000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 22:40:16.816000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 22:40:16.817000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 22:40:16.817000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 22:40:17.159000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 5 (pid: 10354) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:40:16 + host : t-20260510062431-vhgp8-worker-0.t-20260510062431-vhgp8-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank2]:[W509 22:42:04.570219551 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:42:04.674856908 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:42:04.688744613 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:42:04.690226164 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:42:04.708639331 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:42:04.711844858 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:42:04.713163577 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:52:04.686272953 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57250, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:52:04.686354506 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57250, remote=[localhost]:29672) timed out after 600000ms +[rank6]:[W509 22:52:04.700037844 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57216, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:52:04.700099486 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57216, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fe109c87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7fe159c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fe15e2eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fe15e2ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe15e28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe15e28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe15e28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fe10aa6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fe10aa6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7fe10aa73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fe10aa744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fe10aa8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7fe15e27c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7fe15e28bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7fe15d9569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7fe15e29a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7fe15e29b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7fe1664a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7fe165bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7fe1724401ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7fe17244028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f5234d6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f5284d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f52893cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f52893d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f528937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f528937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f528937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f5235b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f5235b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f5235b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f5235b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f5235b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f528936165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f5289370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f5288a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f528937f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f528938076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f529158975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f5290c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f529d55d1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f529d55d28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 22:52:04.792459094 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57194, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:52:04.792525538 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57194, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W509 22:52:04.793034451 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57192, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:52:04.793092737 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57192, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8c44214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f8c94133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8c987cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8c987d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8c9877392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8c9877392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8c9877392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8c44f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8c44f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f8c44f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8c44f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8c44f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f8c9876165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f8c98770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f8c97e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f8c9877f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f8c9878076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f8ca098975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f8ca009911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f8cac9de1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f8cac9de28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3033d2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f3083c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f30882eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f30882ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f308828e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f308828e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f308828e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3034a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3034a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f3034a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3034a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3034a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f308827c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f308828bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f30879569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f308829a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f308829b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f30904a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f308fbb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f309c4ec1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f309c4ec28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:52:04.807288233 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57190, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:52:04.807353378 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57190, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 22:52:04.813854681 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57234, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:52:04.813914149 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57234, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f755b614818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f75ab533c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f75afbcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f75afbd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f75afb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f75afb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f75afb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f755c34717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f755c3532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f755c358cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f755c3594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f755c36913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f75afb6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f75afb70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f75af23b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f75afb7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f75afb8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f75b7d8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f75b749911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f75c3de91ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f75c3de928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:52:04.817715117 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57214, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:52:04.817824853 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57214, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f6608814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f6658733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f665cdcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f665cdd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f665cd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f665cd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f665cd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f660954717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f66095532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f6609558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f66095594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f660956913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f665cd6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f665cd70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f665c43b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f665cd7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f665cd8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f6664f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f666469911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f6670f8d1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f6670f8d28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f0a2f32f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f0a7f24ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f0a838eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f0a838ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0a8388e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0a8388e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0a8388e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f0a3006217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f0a3006e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f0a30073cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f0a300744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f0a3008413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f0a8387c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f0a8388bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f0a82f569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f0a8389a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f0a8389b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f0a8baa475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f0a8b1b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f0a97a711ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f0a97a7128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:52:05.027000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:52:05.028000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 22:52:05.028000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 22:52:05.029000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 22:52:05.243000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 10350) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:52:05 + host : t-20260510064033-d2blz-worker-0.t-20260510064033-d2blz-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10351) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_22:52:05 + host : t-20260510064033-d2blz-worker-0.t-20260510064033-d2blz-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2026-05-09_22:52:05 + host : t-20260510064033-d2blz-worker-0.t-20260510064033-d2blz-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10355) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:52:05 + host : t-20260510064033-d2blz-worker-0.t-20260510064033-d2blz-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10350) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank3]:[W509 22:53:54.456272373 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:53:54.500533147 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:53:54.536185961 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:53:54.618188173 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:53:54.619057736 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:53:54.627731104 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:53:54.628088842 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:03:54.553079867 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44428, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:03:54.553161740 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44428, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W509 23:03:54.565036517 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44410, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:03:54.565099710 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44410, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: if not args.wrap or args.wrap_mode != "stream": +[rank7]: ^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f6333814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f6383733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f6387dcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f6387dd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6387d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6387d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6387d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f633454717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f63345532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f6334558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f63345594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f633456913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f6387d6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f6387d70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f638743b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f6387d7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f6387d8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f638ff8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f638f69911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f639bfdd1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f639bfdd28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: if not args.wrap or args.wrap_mode != "stream": +[rank3]: ^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7faa78414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7faac8333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7faacc9cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7faacc9d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faacc97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faacc97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faacc97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7faa7914717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7faa791532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7faa79158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7faa791594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7faa7916913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7faacc96165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7faacc970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7faacc03b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7faacc97f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7faacc98076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7faad4b8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7faad429911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7faae0bb11ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7faae0bb128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 23:03:54.617384970 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44408, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:03:54.617446861 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44408, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: if not args.wrap or args.wrap_mode != "stream": +[rank4]: ^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fcf38b2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fcf88a4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fcf8d0eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fcf8d0ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcf8d08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcf8d08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcf8d08e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fcf3986217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fcf3986e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fcf39873cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fcf398744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fcf3988413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fcf8d07c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fcf8d08bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fcf8c7569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fcf8d09a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fcf8d09b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fcf952a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fcf949b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fcfa12b11ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fcfa12b128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 23:03:54.643885314 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44390, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:03:54.643949918 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44390, remote=[localhost]:29672) timed out after 600000ms +[rank2]:[W509 23:03:54.644141720 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44380, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:03:54.644205774 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44380, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: if not args.wrap or args.wrap_mode != "stream": +[rank1]: ^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fc790214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fc7e0133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fc7e47cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fc7e47d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc7e477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc7e477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc7e477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fc790f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fc790f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fc790f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fc790f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fc790f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fc7e476165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fc7e4770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fc7e3e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fc7e477f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fc7e478076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fc7ec98975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fc7ec09911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fc7f89a01ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fc7f89a028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: if not args.wrap or args.wrap_mode != "stream": +[rank2]: ^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f5afc414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f5b4c333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f5b509cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f5b509d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5b5097392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5b5097392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5b5097392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f5afd14717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f5afd1532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f5afd158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f5afd1594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f5afd16913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f5b5096165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f5b50970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f5b5003b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f5b5097f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f5b5098076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f5b58b8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f5b5829911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f5b64bb11ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f5b64bb128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:03:54.723050367 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44442, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:03:54.723136619 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44442, remote=[localhost]:29672) timed out after 600000ms +[rank5]:[W509 23:03:54.732044726 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44416, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:03:54.732108776 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44416, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: if not args.wrap or args.wrap_mode != "stream": +[rank6]: ^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8cbc12f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f8d0c04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8d106eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8d106ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8d1068e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8d1068e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8d1068e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8cbce6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8cbce6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f8cbce73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8cbce744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8cbce8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f8d1067c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f8d1068bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f8d0fd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f8d1069a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f8d1069b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f8d188a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f8d17fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f8d248e61ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f8d248e628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: "init_gold_top100": float(masked_topk_acc(bridge.state.detach(), bridge.ids, bridge.corrupt_mask, 100).cpu()), + +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: if not args.wrap or args.wrap_mode != "stream": +[rank5]: ^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f7e4ae87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f7e9ae4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f7e9f4eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f7e9f4ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7e9f48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7e9f48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7e9f48e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f7e4bc6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f7e4bc6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f7e4bc73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f7e4bc744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f7e4bc8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f7e9f47c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f7e9f48bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f7e9eb569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f7e9f49a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f7e9f49b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f7ea76a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f7ea6db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f7eb36161ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f7eb361628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:03:54.951000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:03:54.952000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:03:54.953000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:03:54.953000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 23:03:54.953000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0509 23:03:55.197000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10352) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_23:03:54 + host : t-20260510065222-4nmnk-worker-0.t-20260510065222-4nmnk-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_23:03:54 + host : t-20260510065222-4nmnk-worker-0.t-20260510065222-4nmnk-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:03:54 + host : t-20260510065222-4nmnk-worker-0.t-20260510065222-4nmnk-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10352) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank3]:[W509 23:10:06.977457831 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:10:07.204298771 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:10:07.217871875 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:10:07.388584132 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:10:07.402322800 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:10:07.404039572 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:10:07.404830289 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:20:06.120155829 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:45642, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:20:06.120233693 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:45642, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff5ec214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7ff63c133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff6407cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff6407d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff64077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff64077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff64077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff5ecf4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff5ecf532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7ff5ecf58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff5ecf594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff5ecf6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7ff64076165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7ff640770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7ff63fe3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7ff64077f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7ff64078076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7ff64898975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7ff64809911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7ff6549b71ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7ff6549b728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 23:20:07.280040357 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:45634, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:20:07.280107476 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:45634, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f101016c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f1060133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f10647cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f10647d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f106477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f106477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f106477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f1010f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f1010f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f1010f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f1010f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f1010f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f106476165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f1064770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f1063e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f106477f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f106478076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f106c98975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f106c09911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f10789871ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f107898728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 23:20:07.312911813 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:45600, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:20:07.312979789 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:45600, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fc21fd6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fc26fd33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fc2743cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fc2743d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc27437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc27437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc27437392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fc220b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fc220b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fc220b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fc220b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fc220b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fc27436165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fc274370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fc273a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fc27437f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fc27438076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fc27c58975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fc27bc9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fc2885431ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fc28854328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:20:07.233000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:20:07.233000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:20:07.234000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:20:07.234000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:20:07.235000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 23:20:07.235000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 23:20:07.236000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 23:20:07.564000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10352) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:20:07 + host : t-20260510070411-8wj7l-worker-0.t-20260510070411-8wj7l-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10352) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank1]:[W509 23:21:55.117060875 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:21:55.119806335 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:21:55.163441557 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:21:55.206704865 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:21:56.225520399 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:21:56.235873476 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:21:56.240513156 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:31:55.183293921 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59790, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:31:55.183386448 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59790, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f9ff1f2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7fa041e4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fa0464eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fa0464ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa04648e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa04648e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fa04648e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f9ff2c6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f9ff2c6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f9ff2c73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f9ff2c744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f9ff2c8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7fa04647c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7fa04648bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7fa045b569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7fa04649a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7fa04649b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7fa04e6a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7fa04ddb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7fa05a6d61ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7fa05a6d628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 23:31:56.228117689 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59832, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:31:56.228185554 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59832, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W509 23:31:56.229555627 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59862, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:31:56.229651781 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59862, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f120ed2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f125ec4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f12632eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f12632ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f126328e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f126328e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f126328e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f120fa6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f120fa6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f120fa73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f120fa744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f120fa8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f126327c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f126328bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f12629569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f126329a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f126329b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f126b4a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f126abb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f12774e81ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f12774e828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb459687818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7fb4a964ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb4adceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb4adcebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb4adc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb4adc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb4adc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb45a46217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb45a46e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7fb45a473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb45a4744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb45a48413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7fb4adc7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7fb4adc8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7fb4ad3569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7fb4adc9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7fb4adc9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7fb4b5ea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7fb4b55b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7fb4c1e2e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7fb4c1e2e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 23:31:56.296043360 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59794, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:31:56.296121297 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59794, remote=[localhost]:29672) timed out after 600000ms +[rank7]:[W509 23:31:56.305421950 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59792, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:31:56.305497593 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59792, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fd17072f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7fd1c064ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fd1c4ceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fd1c4cebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd1c4c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd1c4c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd1c4c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fd17146217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fd17146e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7fd171473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fd1714744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fd17148413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7fd1c4c7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7fd1c4c8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7fd1c43569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7fd1c4c9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7fd1c4c9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7fd1ccea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7fd1cc5b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7fd1d8e4a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7fd1d8e4a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff346d2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7ff396c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff39b2eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff39b2ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff39b28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff39b28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff39b28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff347a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff347a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7ff347a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff347a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff347a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7ff39b27c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7ff39b28bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7ff39a9569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7ff39b29a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7ff39b29b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7ff3a34a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7ff3a2bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7ff3af4ca1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7ff3af4ca28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 23:31:56.329053400 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59796, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:31:56.329147823 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59796, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f09a492f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f09f484ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f09f8eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f09f8eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f09f8e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f09f8e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f09f8e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f09a566217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f09a566e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f09a5673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f09a56744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f09a568413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f09f8e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f09f8e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f09f85569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f09f8e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f09f8e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f0a010a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f0a007b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f0a0d0681ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f0a0d06828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:31:56.344952067 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:59834, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:31:56.345031466 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:59834, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fee0a214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7fee5a133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fee5e7cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fee5e7d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fee5e77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fee5e77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fee5e77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fee0af4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fee0af532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7fee0af58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fee0af594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fee0af6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7fee5e76165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7fee5e770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7fee5de3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7fee5e77f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7fee5e78076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7fee6698975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7fee6609911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7fee72a111ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7fee72a1128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:31:56.488000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:31:56.488000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:31:56.489000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:31:56.489000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:31:56.489000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:31:56.490000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 23:31:56.490000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0509 23:31:56.768000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 7 (pid: 10356) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:31:56 + host : t-20260510072023-2v2n8-worker-0.t-20260510072023-2v2n8-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank5]:[W509 23:33:44.160265382 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:33:44.342385238 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:33:44.342456930 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:33:44.380111025 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:33:44.400679135 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:33:44.401831782 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:33:44.421510725 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:43:44.282551245 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37912, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:43:44.282636042 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37912, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f842b814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f847b733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f847fdcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f847fdd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f847fd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f847fd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f847fd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f842c54717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f842c5532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f842c558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f842c5594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f842c56913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f847fd6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f847fd70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f847f43b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f847fd7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f847fd8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f8487f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f848769911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f8493fd91ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f8493fd928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 23:43:44.368712980 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37980, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:43:44.368791807 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37980, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fabb4f2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fac04e4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fac094eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fac094ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fac0948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fac0948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fac0948e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fabb5c6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fabb5c6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fabb5c73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fabb5c744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fabb5c8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fac0947c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fac0948bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fac08b569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fac0949a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fac0949b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fac116a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fac10db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fac1d6801ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fac1d68028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 23:43:44.398839393 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37984, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:43:44.398915101 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37984, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fec2fb6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7fec7fb33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fec841cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fec841d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fec8417392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fec8417392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fec8417392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fec3094717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fec309532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7fec30958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fec309594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fec3096913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7fec8416165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7fec84170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7fec8383b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7fec8417f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7fec8418076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7fec8c38975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7fec8ba9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7fec9831e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7fec9831e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:43:44.447064775 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37918, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:43:44.447141521 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37918, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fecbb214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7fed0b133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fed0f7cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fed0f7d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fed0f77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fed0f77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fed0f77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fecbbf4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fecbbf532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7fecbbf58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fecbbf594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fecbbf6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7fed0f76165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7fed0f770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7fed0ee3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7fed0f77f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7fed0f78076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7fed1798975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7fed1709911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7fed239c81ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7fed239c828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 23:43:44.504045600 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37958, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:43:44.504119734 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37958, remote=[localhost]:29672) timed out after 600000ms +[rank2]:[W509 23:43:44.506322151 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37932, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:43:44.506388392 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37932, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f5d8cc87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f5ddcc4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f5de12eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f5de12ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5de128e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5de128e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f5de128e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f5d8da6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f5d8da6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f5d8da73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f5d8da744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f5d8da8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f5de127c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f5de128bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f5de09569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f5de129a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f5de129b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f5de94a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f5de8bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f5df543d1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f5df543d28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f6106c14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f6156b33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f615b1cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f615b1d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f615b17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f615b17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f615b17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f610794717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f61079532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f6107958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f61079594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f610796913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f615b16165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f615b170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f615a83b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f615b17f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f615b18076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f616338975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f6162a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f616f3951ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f616f39528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 23:43:44.518097326 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:37946, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:43:44.518176745 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:37946, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f2aae72f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f2afe64ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f2b02ceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f2b02cebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2b02c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2b02c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2b02c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f2aaf46217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f2aaf46e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f2aaf473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f2aaf4744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f2aaf48413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f2b02c7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f2b02c8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f2b023569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f2b02c9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f2b02c9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f2b0aea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f2b0a5b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f2b16ef81ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f2b16ef828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:43:44.599000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:43:44.599000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:43:44.600000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:43:44.600000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:43:44.600000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:43:44.601000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 23:43:44.601000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 23:43:44.926000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 5 (pid: 10354) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:43:44 + host : t-20260510073213-gvscg-worker-0.t-20260510073213-gvscg-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank1]:[W509 23:49:41.488217256 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:49:41.691500474 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:49:41.693589382 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:49:41.713876388 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:49:41.715847003 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:49:41.718300058 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:49:41.719255597 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. diff --git a/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_rollout1_p1_bench4gpu_20260513_152806.log b/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_rollout1_p1_bench4gpu_20260513_152806.log new file mode 100644 index 0000000000000000000000000000000000000000..0a1eaaf09a1f9be262db12cb20a67e696ccf487b --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_rollout1_p1_bench4gpu_20260513_152806.log @@ -0,0 +1,103 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8734897", + "vocab_size": 50257, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/lta_owt_gpt2cached_len1024_rollout1_p1_bench4gpu_20260513_152806", + "batch_size": 32, + "grad_accum": 4, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "cosine", + "optimizer": "adamw", + "warmup_steps": 5, + "min_lr": 6e-05, + "weight_decay": 0.1, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "ema_decay": 0.0, + "ema_start_step": 0, + "model_type": "ddit", + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "target_loss": "hard_ce", + "meanflow_weight": 0.0, + "rollout_train_prob": 1.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 8, + "latest_every": 100000, + "resume_path": "" +} +step=5 micro_steps=20 elapsed=17.2s lr=6.000000e-04 loss=10.8125 loss_recon=10.8125 loss_meanflow=0.0000 mean_model_t=0.5068 mean_corrupt_t=0.5068 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 acc_all=0.0005 acc_corrupt=0.0005 corrupt_frac=0.5206 loss_all=10.8125 loss_corrupt=10.8125 acc_corrupt_t_0p0_0p2=0.0011 corrupt_frac_t_0p0_0p2=0.1067 acc_corrupt_t_0p2_0p4=0.0004 corrupt_frac_t_0p2_0p4=0.1344 acc_corrupt_t_0p4_0p6=0.0000 corrupt_frac_t_0p4_0p6=0.0925 acc_corrupt_t_0p6_0p8=0.0007 corrupt_frac_t_0p6_0p8=0.4272 acc_corrupt_t_0p8_1p0=0.0000 corrupt_frac_t_0p8_1p0=0.2392 wrong_frac=0.3916 init_acc_corrupt=0.5891 init_gold_top10=0.6052 init_gold_top100=0.6225 +step=10 micro_steps=40 elapsed=19.8s lr=6.000000e-05 loss=10.8125 loss_recon=10.8125 loss_meanflow=0.0000 mean_model_t=0.4822 mean_corrupt_t=0.4822 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 acc_all=0.0007 acc_corrupt=0.0006 corrupt_frac=0.6584 loss_all=10.8125 loss_corrupt=10.8125 acc_corrupt_t_0p0_0p2=0.0010 corrupt_frac_t_0p0_0p2=0.3402 acc_corrupt_t_0p2_0p4=0.0003 corrupt_frac_t_0p2_0p4=0.1348 acc_corrupt_t_0p4_0p6=0.0005 corrupt_frac_t_0p4_0p6=0.1821 acc_corrupt_t_0p6_0p8=0.0000 corrupt_frac_t_0p6_0p8=0.1046 acc_corrupt_t_0p8_1p0=0.0004 corrupt_frac_t_0p8_1p0=0.2383 wrong_frac=0.5652 init_acc_corrupt=0.4016 init_gold_top10=0.4274 init_gold_top100=0.4691 diff --git a/LTA_openwebtext_dualt/logs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_watcher.log b/LTA_openwebtext_dualt/logs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_watcher.log new file mode 100644 index 0000000000000000000000000000000000000000..7c113b9064cbec198be2c109e5acea164d566b76 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_watcher.log @@ -0,0 +1,458 @@ +[watch-gumbel] run_dir=runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525 +[watch-gumbel] out_base=docs/lta_samples/metrics_20260525/owt_t5_absrope_adaln_Cv_to_2v_mask0p1_1p0_sameT_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525 +[watch-gumbel] interval=10000 max_len=1024 steps=128 c=32100->64200 temp=1.45 top_p=0.95 tau=1.0->0.2 n=128 +[watch-gumbel] 2026-05-25_17:58:57 no ckpt yet +[watch-gumbel] 2026-05-25_17:59:27 no ckpt yet +[watch-gumbel] 2026-05-25_17:59:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:00:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:00:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:01:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:01:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:02:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:02:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:03:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:03:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:04:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:04:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:05:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:05:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:06:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:06:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:07:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:07:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:08:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:08:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:09:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:09:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:10:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:10:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:11:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:11:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:12:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:12:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:13:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:13:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:14:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:14:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:15:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:15:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:16:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:16:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:17:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:17:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:18:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:18:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:19:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:19:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:20:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:20:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:21:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:21:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:22:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:22:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:23:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:23:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:24:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:24:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:25:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:25:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:26:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:26:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:27:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:27:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:28:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:28:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:29:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:29:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:30:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:30:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:31:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:31:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:32:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:32:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:33:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:33:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:34:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:34:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:35:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:35:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:36:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:36:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:37:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:37:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:38:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:38:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:39:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:39:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:40:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:40:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:41:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:41:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:42:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:42:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:43:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:43:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:44:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:44:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:45:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:45:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:46:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:46:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:47:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:47:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:48:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:48:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:49:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:49:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:50:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:50:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:51:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:51:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:52:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:52:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:53:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:53:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:54:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:54:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:55:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:55:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:56:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:56:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:57:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:57:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:58:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:58:57 no ckpt yet +[watch-gumbel] 2026-05-25_18:59:27 no ckpt yet +[watch-gumbel] 2026-05-25_18:59:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:00:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:00:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:01:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:01:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:02:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:02:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:03:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:03:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:04:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:04:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:05:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:05:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:06:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:06:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:07:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:07:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:08:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:08:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:09:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:09:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:10:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:10:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:11:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:11:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:12:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:12:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:13:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:13:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:14:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:14:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:15:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:15:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:16:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:16:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:17:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:17:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:18:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:18:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:19:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:19:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:20:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:20:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:21:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:21:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:22:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:22:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:23:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:23:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:24:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:24:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:25:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:25:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:26:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:26:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:27:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:27:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:28:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:28:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:29:27 no ckpt yet +[watch-gumbel] 2026-05-25_19:29:57 no ckpt yet +[watch-gumbel] 2026-05-25_19:30:27 infer runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000.pt -> docs/lta_samples/metrics_20260525/owt_t5_absrope_adaln_Cv_to_2v_mask0p1_1p0_sameT_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000 +[load] runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000.pt +[ckpt] step=10000 +[sde] generated 2/128 +[sde] generated 4/128 +[sde] generated 6/128 +[sde] generated 8/128 +[sde] generated 10/128 +[sde] generated 12/128 +[sde] generated 14/128 +[sde] generated 16/128 +[sde] generated 18/128 +[sde] generated 20/128 +[sde] generated 22/128 +[sde] generated 24/128 +[sde] generated 26/128 +[sde] generated 28/128 +[sde] generated 30/128 +[sde] generated 32/128 +[sde] generated 34/128 +[sde] generated 36/128 +[sde] generated 38/128 +[sde] generated 40/128 +[sde] generated 42/128 +[sde] generated 44/128 +[sde] generated 46/128 +[sde] generated 48/128 +[sde] generated 50/128 +[sde] generated 52/128 +[sde] generated 54/128 +[sde] generated 56/128 +[sde] generated 58/128 +[sde] generated 60/128 +[sde] generated 62/128 +[sde] generated 64/128 +[sde] generated 66/128 +[sde] generated 68/128 +[sde] generated 70/128 +[sde] generated 72/128 +[sde] generated 74/128 +[sde] generated 76/128 +[sde] generated 78/128 +[sde] generated 80/128 +[sde] generated 82/128 +[sde] generated 84/128 +[sde] generated 86/128 +[sde] generated 88/128 +[sde] generated 90/128 +[sde] generated 92/128 +[sde] generated 94/128 +[sde] generated 96/128 +[sde] generated 98/128 +[sde] generated 100/128 +[sde] generated 102/128 +[sde] generated 104/128 +[sde] generated 106/128 +[sde] generated 108/128 +[sde] generated 110/128 +[sde] generated 112/128 +[sde] generated 114/128 +[sde] generated 116/128 +[sde] generated 118/128 +[sde] generated 120/128 +[sde] generated 122/128 +[sde] generated 124/128 +[sde] generated 126/128 +[sde] generated 128/128 +[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard +[summary] { + "type": "summary", + "checkpoint": "runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000.pt", + "step": 10000, + "decode": { + "decode_rule": "dirichlet_resample_sde", + "steps": 128, + "model_t_mode": "support_t", + "mean_mode": "endpoint_only", + "anchor_gamma": 1.0, + "endpoint_floor": 0.0, + "concentration_min": 32100.0, + "concentration_max": 64200.0, + "endpoint_temp": 1.45, + "endpoint_temp_start": null, + "endpoint_temp_end": null, + "endpoint_projection": "gumbel_softmax", + "endpoint_top_k": 0, + "endpoint_top_p": 0.95, + "gumbel_tau_start": 1.0, + "gumbel_tau_end": 0.2, + "gumbel_noise_scale_start": 1.0, + "gumbel_noise_scale_end": 1.0, + "ban_special_tokens": false, + "banned_endpoint_ids": [], + "support_power": 1.0, + "semantic_power": 1.0, + "noise_init": "dirichlet", + "noise_sigma": -1.0, + "noise_dirichlet_concentration": 32100.0, + "sde_resample": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 3.0, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.0, + "final_from": "blend_0.5", + "n_samples": 128, + "seed": 20260524 + }, + "raw_genppl": { + "ppl": 2.098817389847081, + "nll_per_token": 0.7413740384102933, + "tokens": 129915, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "stripped_genppl": { + "ppl": 2.091704636784785, + "nll_per_token": 0.737979349229492, + "tokens": 129874, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "diversity": { + "sample_entropy": 1.1927971824809422, + "unique_tokens": 450, + "token_count": 131072, + "distinct_1": 0.0034332275390625, + "distinct_2": 0.02113880742913001, + "top_token_mass": 0.6524658203125 + } +} +[done] docs/lta_samples/metrics_20260525/owt_t5_absrope_adaln_Cv_to_2v_mask0p1_1p0_sameT_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000/sde_steps128_samples128_scored.jsonl +[watch-gumbel] 2026-05-25_19:37:00 done step_0010000 +[watch-gumbel] 2026-05-25_21:02:31 infer runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000.pt -> docs/lta_samples/metrics_20260525/owt_t5_absrope_adaln_Cv_to_2v_mask0p1_1p0_sameT_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000 +[load] runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000.pt +[ckpt] step=20000 +[sde] generated 2/128 +[sde] generated 4/128 +[sde] generated 6/128 +[sde] generated 8/128 +[sde] generated 10/128 +[sde] generated 12/128 +[sde] generated 14/128 +[sde] generated 16/128 +[sde] generated 18/128 +[sde] generated 20/128 +[sde] generated 22/128 +[sde] generated 24/128 +[sde] generated 26/128 +[sde] generated 28/128 +[sde] generated 30/128 +[sde] generated 32/128 +[sde] generated 34/128 +[sde] generated 36/128 +[sde] generated 38/128 +[sde] generated 40/128 +[sde] generated 42/128 +[sde] generated 44/128 +[sde] generated 46/128 +[sde] generated 48/128 +[sde] generated 50/128 +[sde] generated 52/128 +[sde] generated 54/128 +[sde] generated 56/128 +[sde] generated 58/128 +[sde] generated 60/128 +[sde] generated 62/128 +[sde] generated 64/128 +[sde] generated 66/128 +[sde] generated 68/128 +[sde] generated 70/128 +[sde] generated 72/128 +[sde] generated 74/128 +[sde] generated 76/128 +[sde] generated 78/128 +[sde] generated 80/128 +[sde] generated 82/128 +[sde] generated 84/128 +[sde] generated 86/128 +[sde] generated 88/128 +[sde] generated 90/128 +[sde] generated 92/128 +[sde] generated 94/128 +[sde] generated 96/128 +[sde] generated 98/128 +[sde] generated 100/128 +[sde] generated 102/128 +[sde] generated 104/128 +[sde] generated 106/128 +[sde] generated 108/128 +[sde] generated 110/128 +[sde] generated 112/128 +[sde] generated 114/128 +[sde] generated 116/128 +[sde] generated 118/128 +[sde] generated 120/128 +[sde] generated 122/128 +[sde] generated 124/128 +[sde] generated 126/128 +[sde] generated 128/128 +[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard +[summary] { + "type": "summary", + "checkpoint": "runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000.pt", + "step": 20000, + "decode": { + "decode_rule": "dirichlet_resample_sde", + "steps": 128, + "model_t_mode": "support_t", + "mean_mode": "endpoint_only", + "anchor_gamma": 1.0, + "endpoint_floor": 0.0, + "concentration_min": 32100.0, + "concentration_max": 64200.0, + "endpoint_temp": 1.45, + "endpoint_temp_start": null, + "endpoint_temp_end": null, + "endpoint_projection": "gumbel_softmax", + "endpoint_top_k": 0, + "endpoint_top_p": 0.95, + "gumbel_tau_start": 1.0, + "gumbel_tau_end": 0.2, + "gumbel_noise_scale_start": 1.0, + "gumbel_noise_scale_end": 1.0, + "ban_special_tokens": false, + "banned_endpoint_ids": [], + "support_power": 1.0, + "semantic_power": 1.0, + "noise_init": "dirichlet", + "noise_sigma": -1.0, + "noise_dirichlet_concentration": 32100.0, + "sde_resample": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 3.0, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.0, + "final_from": "blend_0.5", + "n_samples": 128, + "seed": 20260524 + }, + "raw_genppl": { + "ppl": 3.4360435319768396, + "nll_per_token": 1.234320673418016, + "tokens": 60733, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "stripped_genppl": { + "ppl": 3.399305871374786, + "nll_per_token": 1.2235712553015452, + "tokens": 60637, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "diversity": { + "sample_entropy": 0.8467485464533029, + "unique_tokens": 298, + "token_count": 131072, + "distinct_1": 0.0022735595703125, + "distinct_2": 0.017037817693059627, + "top_token_mass": 0.5230484008789062 + } +} +[done] docs/lta_samples/metrics_20260525/owt_t5_absrope_adaln_Cv_to_2v_mask0p1_1p0_sameT_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000/sde_steps128_samples128_scored.jsonl +[watch-gumbel] 2026-05-25_21:08:53 done step_0020000 diff --git a/LTA_openwebtext_dualt/logs/owt_candidate_catdualt_step246k_64_c1024_t1p2_blend_n64.log b/LTA_openwebtext_dualt/logs/owt_candidate_catdualt_step246k_64_c1024_t1p2_blend_n64.log new file mode 100644 index 0000000000000000000000000000000000000000..15dee0541d9c50bde49573030b7b1a1d0e12bae5 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/owt_candidate_catdualt_step246k_64_c1024_t1p2_blend_n64.log @@ -0,0 +1,3 @@ +[forbid_endpoint_ids] n=352 first=[94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125] +[decode] steps64_c1024_mtpost_t1p2_tpow1p0_noise0_blend_anchored +[summary] {"name": "steps64_c1024_mtpost_t1p2_tpow1p0_noise0_blend_anchored", "step": 246000, "n_samples": 64, "steps": 64, "concentration_max": 1024.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "update_rule": "anchored", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": 153.70967053904752, "sample_entropy": 4.642808948434547, "distinct_1": 0.1154937744140625, "distinct_2": 0.5061858504398827, "top_token_mass": 0.1103973388671875, "tokens_scored": 59240, "readability_score": 4.950578398852453, "mean_chars": 3752.34375, "replacement_chars": 0.0} diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/smoke_gpt2_softendpoint_mn_n128_onehot.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/smoke_gpt2_softendpoint_mn_n128_onehot.log new file mode 100644 index 0000000000000000000000000000000000000000..0955d2c42793498e2c195f63f5433f5426ffdb27 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/smoke_gpt2_softendpoint_mn_n128_onehot.log @@ -0,0 +1,94 @@ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2600, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1866, in main +[rank0]: dataset = CachedWrappedTextSequenceDataset( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/data.py", line 704, in __init__ +[rank0]: raise ValueError(f"cache max_len={cache_max_len} does not match requested max_len={self.max_len}") +[rank0]: ValueError: cache max_len=1024 does not match requested max_len=128 +[rank0]:[W516 22:06:53.301173242 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0516 22:06:53.771000 470531 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470536 closing signal SIGTERM +W0516 22:06:53.772000 470531 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470537 closing signal SIGTERM +W0516 22:06:53.773000 470531 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470538 closing signal SIGTERM +E0516 22:06:53.950000 470531 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 470535) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-16_22:06:53 + host : localhost + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 470535) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2600, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1866, in main +[rank0]: dataset = CachedWrappedTextSequenceDataset( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/data.py", line 695, in __init__ +[rank0]: raise RuntimeError( +[rank0]: RuntimeError: cached OWT chunks not found under /e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len128_train_minus_100k; build them first or set --owt_chunk_cache_rebuild on rank 0 +[rank0]:[W516 22:07:24.000487031 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0516 22:07:24.408000 470659 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470664 closing signal SIGTERM +W0516 22:07:24.409000 470659 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470665 closing signal SIGTERM +W0516 22:07:24.409000 470659 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 470666 closing signal SIGTERM +E0516 22:07:24.587000 470659 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 470663) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-16_22:07:24 + host : localhost + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 470663) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_dirichlet_unigram_shared_highC_20260517_170456.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_dirichlet_unigram_shared_highC_20260517_170456.log new file mode 100644 index 0000000000000000000000000000000000000000..d54081b18099535c76486967b11dea788104a499 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_dirichlet_unigram_shared_highC_20260517_170456.log @@ -0,0 +1,197 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_combo_len256_dirichlet_unigram_shared_highC_20260517_170456", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.7s lr=2.000000e-03 loss=6.6629 loss_recon=6.6629 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1701 corrupt_frac=1.0000 acc_corrupt=0.1701 loss_corrupt=6.6629 wrong_frac=0.7922 init_acc_corrupt=0.2054 acc_corrupt_t_0p0_0p2=0.0802 corrupt_frac_t_0p0_0p2=0.5620 acc_corrupt_t_0p2_0p4=0.2399 corrupt_frac_t_0p2_0p4=0.3533 acc_corrupt_t_0p4_0p6=0.4568 corrupt_frac_t_0p4_0p6=0.0755 acc_corrupt_t_0p6_0p8=0.6274 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=1.2395 out_g_norm=0.9648 acc_corrupt_t_0p8_1p0=0.9141 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.3374 init_gold_top10=0.2275 init_gold_top100=0.2997 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=4.0s lr=2.000000e-03 loss=5.9494 loss_recon=5.9494 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1795 corrupt_frac=1.0000 acc_corrupt=0.1795 loss_corrupt=5.9494 wrong_frac=0.7916 init_acc_corrupt=0.2060 acc_corrupt_t_0p0_0p2=0.0927 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.2447 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.4631 corrupt_frac_t_0p4_0p6=0.0744 acc_corrupt_t_0p6_0p8=0.6568 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.8105 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=4.1195 out_g_norm=1.3169 loss_all=5.5816 init_gold_top10=0.2191 init_gold_top100=0.2940 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=4.0s lr=2.000000e-03 loss=5.3088 loss_recon=5.3088 loss_meanflow=0.0000 mean_model_t=0.2105 mean_corrupt_t=0.2105 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2092 corrupt_frac=1.0000 acc_corrupt=0.2092 loss_corrupt=5.3088 wrong_frac=0.7894 init_acc_corrupt=0.2082 acc_corrupt_t_0p0_0p2=0.1143 corrupt_frac_t_0p0_0p2=0.5542 acc_corrupt_t_0p2_0p4=0.2854 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.4737 corrupt_frac_t_0p4_0p6=0.0813 acc_corrupt_t_0p6_0p8=0.6627 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=6.7925 out_g_norm=0.5991 acc_corrupt_t_0p8_1p0=0.8750 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.0754 init_gold_top10=0.2283 init_gold_top100=0.3011 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=4.0s lr=2.000000e-03 loss=5.0142 loss_recon=5.0142 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2180 corrupt_frac=1.0000 acc_corrupt=0.2180 loss_corrupt=5.0142 wrong_frac=0.7915 init_acc_corrupt=0.2061 acc_corrupt_t_0p0_0p2=0.1264 corrupt_frac_t_0p0_0p2=0.5597 acc_corrupt_t_0p2_0p4=0.2952 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.4799 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.6613 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=8.4766 out_g_norm=0.3326 acc_corrupt_t_0p8_1p0=0.7930 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7706 init_gold_top10=0.2298 init_gold_top100=0.3036 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=4.0s lr=2.000000e-03 loss=4.5487 loss_recon=4.5487 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2257 corrupt_frac=1.0000 acc_corrupt=0.2257 loss_corrupt=4.5487 wrong_frac=0.7914 init_acc_corrupt=0.2062 acc_corrupt_t_0p0_0p2=0.1340 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.3049 corrupt_frac_t_0p2_0p4=0.3550 acc_corrupt_t_0p4_0p6=0.4866 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.6614 corrupt_frac_t_0p6_0p8=0.0143 out_w_norm=9.7624 out_g_norm=0.4423 loss_all=4.2801 init_gold_top10=0.2068 init_gold_top100=0.2830 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=4.0s lr=2.000000e-03 loss=3.8447 loss_recon=3.8447 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2370 corrupt_frac=1.0000 acc_corrupt=0.2370 loss_corrupt=3.8447 wrong_frac=0.7917 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.1444 corrupt_frac_t_0p0_0p2=0.5568 acc_corrupt_t_0p2_0p4=0.3165 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.4950 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.6750 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=10.1909 out_g_norm=0.4588 loss_all=3.5868 init_gold_top10=0.1960 init_gold_top100=0.2722 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=4.0s lr=2.000000e-03 loss=3.1110 loss_recon=3.1110 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2652 corrupt_frac=1.0000 acc_corrupt=0.2652 loss_corrupt=3.1110 wrong_frac=0.7925 init_acc_corrupt=0.2051 acc_corrupt_t_0p0_0p2=0.1631 corrupt_frac_t_0p0_0p2=0.5607 acc_corrupt_t_0p2_0p4=0.3575 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.5431 corrupt_frac_t_0p4_0p6=0.0739 acc_corrupt_t_0p6_0p8=0.7116 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=10.4454 out_g_norm=0.5395 loss_all=2.6425 init_gold_top10=0.2289 init_gold_top100=0.3008 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=4.0s lr=2.000000e-03 loss=2.2335 loss_recon=2.2335 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3921 corrupt_frac=1.0000 acc_corrupt=0.3921 loss_corrupt=2.2335 wrong_frac=0.7904 init_acc_corrupt=0.2072 acc_corrupt_t_0p0_0p2=0.2394 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.5444 corrupt_frac_t_0p2_0p4=0.3649 acc_corrupt_t_0p4_0p6=0.7185 corrupt_frac_t_0p4_0p6=0.0765 out_w_norm=10.7231 out_g_norm=0.6774 acc_corrupt_t_0p6_0p8=0.8264 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=0.9297 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.8154 init_gold_top10=0.2095 init_gold_top100=0.2845 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=4.0s lr=2.000000e-03 loss=1.4273 loss_recon=1.4273 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6031 corrupt_frac=1.0000 acc_corrupt=0.6031 loss_corrupt=1.4273 wrong_frac=0.7901 init_acc_corrupt=0.2076 acc_corrupt_t_0p0_0p2=0.4064 corrupt_frac_t_0p0_0p2=0.5513 acc_corrupt_t_0p2_0p4=0.8248 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.9279 corrupt_frac_t_0p4_0p6=0.0755 out_w_norm=10.9722 out_g_norm=0.6506 acc_corrupt_t_0p6_0p8=0.9541 corrupt_frac_t_0p6_0p8=0.0127 acc_corrupt_t_0p8_1p0=0.9629 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1917 init_gold_top10=0.2077 init_gold_top100=0.2817 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=4.0s lr=2.000000e-03 loss=0.9388 loss_recon=0.9388 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7497 corrupt_frac=1.0000 acc_corrupt=0.7497 loss_corrupt=0.9388 wrong_frac=0.7897 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.5762 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.9639 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.9922 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.2383 out_g_norm=0.6494 acc_corrupt_t_0p6_0p8=0.9947 corrupt_frac_t_0p6_0p8=0.0133 loss_all=0.7538 init_gold_top10=0.2115 init_gold_top100=0.2852 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456.log new file mode 100644 index 0000000000000000000000000000000000000000..d5a1a449024ababd74a5d7322877e3205a178de2 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456.log @@ -0,0 +1,395 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.5, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.03, + "logistic_normal_sigma_max": 0.4, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.5s lr=2.000000e-03 loss=6.7097 loss_recon=6.7097 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1179 corrupt_frac=1.0000 acc_corrupt=0.1179 loss_corrupt=6.7097 wrong_frac=0.8656 init_acc_corrupt=0.1344 acc_corrupt_t_0p0_0p2=0.0569 corrupt_frac_t_0p0_0p2=0.5599 acc_corrupt_t_0p2_0p4=0.1555 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.3428 corrupt_frac_t_0p4_0p6=0.0738 acc_corrupt_t_0p6_0p8=0.5361 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=1.2263 out_g_norm=1.0547 acc_corrupt_t_0p8_1p0=0.8340 corrupt_frac_t_0p8_1p0=0.0104 loss_all=6.4790 init_gold_top10=0.1401 init_gold_top100=0.2195 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=3.8s lr=2.000000e-03 loss=6.1757 loss_recon=6.1757 loss_meanflow=0.0000 mean_model_t=0.2069 mean_corrupt_t=0.2069 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1118 corrupt_frac=1.0000 acc_corrupt=0.1118 loss_corrupt=6.1757 wrong_frac=0.8695 init_acc_corrupt=0.1305 acc_corrupt_t_0p0_0p2=0.0654 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.1396 corrupt_frac_t_0p2_0p4=0.3552 acc_corrupt_t_0p4_0p6=0.2837 corrupt_frac_t_0p4_0p6=0.0749 out_w_norm=3.8017 out_g_norm=1.5027 acc_corrupt_t_0p6_0p8=0.5003 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.7578 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.9357 init_gold_top10=0.1209 init_gold_top100=0.2034 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=3.8s lr=2.000000e-03 loss=5.6465 loss_recon=5.6465 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1395 corrupt_frac=1.0000 acc_corrupt=0.1395 loss_corrupt=5.6465 wrong_frac=0.8662 init_acc_corrupt=0.1338 acc_corrupt_t_0p0_0p2=0.0788 corrupt_frac_t_0p0_0p2=0.5585 acc_corrupt_t_0p2_0p4=0.1795 corrupt_frac_t_0p2_0p4=0.3541 acc_corrupt_t_0p4_0p6=0.3465 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.5308 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=6.0405 out_g_norm=0.6592 acc_corrupt_t_0p8_1p0=0.4082 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.4633 init_gold_top10=0.1470 init_gold_top100=0.2274 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=3.9s lr=2.000000e-03 loss=5.4440 loss_recon=5.4440 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1499 corrupt_frac=1.0000 acc_corrupt=0.1499 loss_corrupt=5.4440 wrong_frac=0.8670 init_acc_corrupt=0.1330 acc_corrupt_t_0p0_0p2=0.0880 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=0.1937 corrupt_frac_t_0p2_0p4=0.3638 acc_corrupt_t_0p4_0p6=0.3606 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.4963 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=7.7267 out_g_norm=0.3063 acc_corrupt_t_0p8_1p0=0.7578 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.2757 init_gold_top10=0.1559 init_gold_top100=0.2377 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=3.8s lr=2.000000e-03 loss=5.1483 loss_recon=5.1483 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1601 corrupt_frac=1.0000 acc_corrupt=0.1601 loss_corrupt=5.1483 wrong_frac=0.8648 init_acc_corrupt=0.1352 acc_corrupt_t_0p0_0p2=0.0953 corrupt_frac_t_0p0_0p2=0.5508 acc_corrupt_t_0p2_0p4=0.2020 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.3747 corrupt_frac_t_0p4_0p6=0.0818 out_w_norm=8.9678 out_g_norm=0.4350 acc_corrupt_t_0p6_0p8=0.5439 corrupt_frac_t_0p6_0p8=0.0136 acc_corrupt_t_0p8_1p0=0.8184 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.0288 init_gold_top10=0.1219 init_gold_top100=0.2024 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.8s lr=2.000000e-03 loss=4.6281 loss_recon=4.6281 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1673 corrupt_frac=1.0000 acc_corrupt=0.1673 loss_corrupt=4.6281 wrong_frac=0.8662 init_acc_corrupt=0.1338 acc_corrupt_t_0p0_0p2=0.1028 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.2130 corrupt_frac_t_0p2_0p4=0.3509 acc_corrupt_t_0p4_0p6=0.3788 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p6_0p8=0.5652 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=9.3953 out_g_norm=0.4892 acc_corrupt_t_0p8_1p0=0.8623 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.4779 init_gold_top10=0.1371 init_gold_top100=0.2186 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.8s lr=2.000000e-03 loss=4.0407 loss_recon=4.0407 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1789 corrupt_frac=1.0000 acc_corrupt=0.1789 loss_corrupt=4.0407 wrong_frac=0.8641 init_acc_corrupt=0.1359 acc_corrupt_t_0p0_0p2=0.1138 corrupt_frac_t_0p0_0p2=0.5534 acc_corrupt_t_0p2_0p4=0.2271 corrupt_frac_t_0p2_0p4=0.3573 acc_corrupt_t_0p4_0p6=0.3743 corrupt_frac_t_0p4_0p6=0.0812 out_w_norm=9.6182 out_g_norm=0.5365 acc_corrupt_t_0p6_0p8=0.5301 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=0.8301 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.7657 init_gold_top10=0.1297 init_gold_top100=0.2126 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.8s lr=2.000000e-03 loss=3.5950 loss_recon=3.5950 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1916 corrupt_frac=1.0000 acc_corrupt=0.1916 loss_corrupt=3.5950 wrong_frac=0.8684 init_acc_corrupt=0.1316 acc_corrupt_t_0p0_0p2=0.1264 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.2418 corrupt_frac_t_0p2_0p4=0.3614 acc_corrupt_t_0p4_0p6=0.3942 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.5762 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=9.8701 out_g_norm=0.7488 acc_corrupt_t_0p8_1p0=0.8203 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.4492 init_gold_top10=0.1366 init_gold_top100=0.2184 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.8s lr=2.000000e-03 loss=3.0683 loss_recon=3.0683 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2626 corrupt_frac=1.0000 acc_corrupt=0.2626 loss_corrupt=3.0683 wrong_frac=0.8669 init_acc_corrupt=0.1331 acc_corrupt_t_0p0_0p2=0.1752 corrupt_frac_t_0p0_0p2=0.5614 acc_corrupt_t_0p2_0p4=0.3404 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=0.5038 corrupt_frac_t_0p4_0p6=0.0757 out_w_norm=10.0897 out_g_norm=1.0269 acc_corrupt_t_0p6_0p8=0.6538 corrupt_frac_t_0p6_0p8=0.0121 acc_corrupt_t_0p8_1p0=0.9082 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.4332 init_gold_top10=0.1616 init_gold_top100=0.2392 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.8s lr=2.000000e-03 loss=2.4691 loss_recon=2.4691 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4023 corrupt_frac=1.0000 acc_corrupt=0.4023 loss_corrupt=2.4691 wrong_frac=0.8652 init_acc_corrupt=0.1348 acc_corrupt_t_0p0_0p2=0.2939 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.5031 corrupt_frac_t_0p2_0p4=0.3610 acc_corrupt_t_0p4_0p6=0.6783 corrupt_frac_t_0p4_0p6=0.0730 out_w_norm=10.3143 out_g_norm=1.0213 acc_corrupt_t_0p6_0p8=0.7848 corrupt_frac_t_0p6_0p8=0.0140 acc_corrupt_t_0p8_1p0=0.4844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.1232 init_gold_top10=0.1450 init_gold_top100=0.2272 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.5, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.03, + "logistic_normal_sigma_max": 0.4, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_combo_len256_logistic_unigram_shared_highC_seqrand_20260517_170456/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=4.7s lr=2.000000e-03 loss=2.0475 loss_recon=2.0475 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5133 corrupt_frac=1.0000 acc_corrupt=0.5133 loss_corrupt=2.0475 wrong_frac=0.8656 init_acc_corrupt=0.1344 acc_corrupt_t_0p0_0p2=0.4118 corrupt_frac_t_0p0_0p2=0.5599 acc_corrupt_t_0p2_0p4=0.6173 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7417 corrupt_frac_t_0p4_0p6=0.0738 acc_corrupt_t_0p6_0p8=0.8054 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=10.5718 out_g_norm=1.1653 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0104 loss_all=1.8965 init_gold_top10=0.1401 init_gold_top100=0.2195 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=4.0s lr=2.000000e-03 loss=1.8251 loss_recon=1.8251 loss_meanflow=0.0000 mean_model_t=0.2069 mean_corrupt_t=0.2069 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5569 corrupt_frac=1.0000 acc_corrupt=0.5569 loss_corrupt=1.8251 wrong_frac=0.8695 init_acc_corrupt=0.1305 acc_corrupt_t_0p0_0p2=0.4760 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.6412 corrupt_frac_t_0p2_0p4=0.3552 acc_corrupt_t_0p4_0p6=0.7336 corrupt_frac_t_0p4_0p6=0.0749 out_w_norm=10.7986 out_g_norm=1.1972 acc_corrupt_t_0p6_0p8=0.8191 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.8619 init_gold_top10=0.1209 init_gold_top100=0.2034 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=4.0s lr=2.000000e-03 loss=1.6495 loss_recon=1.6495 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5851 corrupt_frac=1.0000 acc_corrupt=0.5851 loss_corrupt=1.6495 wrong_frac=0.8662 init_acc_corrupt=0.1338 acc_corrupt_t_0p0_0p2=0.5137 corrupt_frac_t_0p0_0p2=0.5585 acc_corrupt_t_0p2_0p4=0.6553 corrupt_frac_t_0p2_0p4=0.3541 acc_corrupt_t_0p4_0p6=0.7488 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8325 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=10.9370 out_g_norm=1.2443 acc_corrupt_t_0p8_1p0=0.5098 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.5011 init_gold_top10=0.1470 init_gold_top100=0.2274 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=4.0s lr=2.000000e-03 loss=1.4723 loss_recon=1.4723 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6098 corrupt_frac=1.0000 acc_corrupt=0.6098 loss_corrupt=1.4723 wrong_frac=0.8670 init_acc_corrupt=0.1330 acc_corrupt_t_0p0_0p2=0.5484 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=0.6697 corrupt_frac_t_0p2_0p4=0.3638 acc_corrupt_t_0p4_0p6=0.7569 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.7629 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=11.0332 out_g_norm=1.2546 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0527 init_gold_top10=0.1559 init_gold_top100=0.2377 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=4.0s lr=2.000000e-03 loss=1.3097 loss_recon=1.3097 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6239 corrupt_frac=1.0000 acc_corrupt=0.6239 loss_corrupt=1.3097 wrong_frac=0.8648 init_acc_corrupt=0.1352 acc_corrupt_t_0p0_0p2=0.5615 corrupt_frac_t_0p0_0p2=0.5508 acc_corrupt_t_0p2_0p4=0.6788 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7833 corrupt_frac_t_0p4_0p6=0.0818 out_w_norm=11.1027 out_g_norm=1.2651 acc_corrupt_t_0p6_0p8=0.8208 corrupt_frac_t_0p6_0p8=0.0136 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4609 init_gold_top10=0.1219 init_gold_top100=0.2024 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=4.0s lr=2.000000e-03 loss=1.1768 loss_recon=1.1768 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6383 corrupt_frac=1.0000 acc_corrupt=0.6383 loss_corrupt=1.1768 wrong_frac=0.8662 init_acc_corrupt=0.1338 acc_corrupt_t_0p0_0p2=0.5824 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.6902 corrupt_frac_t_0p2_0p4=0.3509 acc_corrupt_t_0p4_0p6=0.7812 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p6_0p8=0.8484 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=11.1630 out_g_norm=1.2914 acc_corrupt_t_0p8_1p0=0.9971 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3553 init_gold_top10=0.1371 init_gold_top100=0.2186 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=4.0s lr=2.000000e-03 loss=1.0551 loss_recon=1.0551 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6547 corrupt_frac=1.0000 acc_corrupt=0.6547 loss_corrupt=1.0551 wrong_frac=0.8641 init_acc_corrupt=0.1359 acc_corrupt_t_0p0_0p2=0.6084 corrupt_frac_t_0p0_0p2=0.5534 acc_corrupt_t_0p2_0p4=0.6984 corrupt_frac_t_0p2_0p4=0.3573 acc_corrupt_t_0p4_0p6=0.7629 corrupt_frac_t_0p4_0p6=0.0812 out_w_norm=11.2082 out_g_norm=1.2603 acc_corrupt_t_0p6_0p8=0.7996 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0073 init_gold_top10=0.1297 init_gold_top100=0.2126 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=4.0s lr=2.000000e-03 loss=0.9843 loss_recon=0.9843 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6577 corrupt_frac=1.0000 acc_corrupt=0.6577 loss_corrupt=0.9843 wrong_frac=0.8684 init_acc_corrupt=0.1316 acc_corrupt_t_0p0_0p2=0.6154 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.6959 corrupt_frac_t_0p2_0p4=0.3614 acc_corrupt_t_0p4_0p6=0.7686 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.8488 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=11.2253 out_g_norm=1.1781 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0908 init_gold_top10=0.1366 init_gold_top100=0.2184 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=4.0s lr=2.000000e-03 loss=0.8950 loss_recon=0.8950 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6766 corrupt_frac=1.0000 acc_corrupt=0.6766 loss_corrupt=0.8950 wrong_frac=0.8669 init_acc_corrupt=0.1331 acc_corrupt_t_0p0_0p2=0.6318 corrupt_frac_t_0p0_0p2=0.5614 acc_corrupt_t_0p2_0p4=0.7194 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=0.7877 corrupt_frac_t_0p4_0p6=0.0757 out_w_norm=11.2363 out_g_norm=1.1321 acc_corrupt_t_0p6_0p8=0.8710 corrupt_frac_t_0p6_0p8=0.0121 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6929 init_gold_top10=0.1616 init_gold_top100=0.2392 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=4.0s lr=2.000000e-03 loss=0.8544 loss_recon=0.8544 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6830 corrupt_frac=1.0000 acc_corrupt=0.6830 loss_corrupt=0.8544 wrong_frac=0.8652 init_acc_corrupt=0.1348 acc_corrupt_t_0p0_0p2=0.6393 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.7180 corrupt_frac_t_0p2_0p4=0.3610 acc_corrupt_t_0p4_0p6=0.8173 corrupt_frac_t_0p4_0p6=0.0730 out_w_norm=11.2470 out_g_norm=1.0160 acc_corrupt_t_0p6_0p8=0.8743 corrupt_frac_t_0p6_0p8=0.0140 acc_corrupt_t_0p8_1p0=0.5039 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7242 init_gold_top10=0.1450 init_gold_top100=0.2272 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_20260517_223933.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_20260517_223933.log new file mode 100644 index 0000000000000000000000000000000000000000..60ca3aaba103f4ae3c005fc3246b8448ba4c183d --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_20260517_223933.log @@ -0,0 +1,229 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_20260517_223933", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +W0517 22:40:01.897000 386925 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0517 22:40:01.899000 386925 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 386929 closing signal SIGTERM +W0517 22:40:01.900000 386925 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 386930 closing signal SIGTERM +W0517 22:40:01.900000 386925 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 386931 closing signal SIGTERM +W0517 22:40:01.901000 386925 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 386932 closing signal SIGTERM +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 386925 got signal: 15 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139.log new file mode 100644 index 0000000000000000000000000000000000000000..b2178ddd71c2613dfb7b0c98ee252f9ed0fac08a --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139.log @@ -0,0 +1,609 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=24.8s lr=2.000000e-03 loss=7.7206 loss_recon=7.7206 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0958 corrupt_frac=1.0000 acc_corrupt=0.0958 loss_corrupt=7.7206 wrong_frac=0.7915 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.0500 corrupt_frac_t_0p0_0p2=0.5640 acc_corrupt_t_0p2_0p4=0.1270 corrupt_frac_t_0p2_0p4=0.3466 acc_corrupt_t_0p4_0p6=0.2493 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p6_0p8=0.3719 corrupt_frac_t_0p6_0p8=0.0136 out_w_norm=1.0047 out_g_norm=1.0928 acc_corrupt_t_0p8_1p0=0.4936 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.4724 init_gold_top10=0.2003 init_gold_top100=0.4085 rollout_applied_pos_frac=0.4453 init_acc_rollout_applied=0.1056 init_acc_rollout_kept=0.1192 logit_acc_rollout_applied=0.0969 logit_acc_rollout_kept=0.0996 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=23.9s lr=2.000000e-03 loss=7.0874 loss_recon=7.0874 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4995 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1036 corrupt_frac=1.0000 acc_corrupt=0.1036 loss_corrupt=7.0874 wrong_frac=0.7905 init_acc_corrupt=0.1172 acc_corrupt_t_0p0_0p2=0.0560 corrupt_frac_t_0p0_0p2=0.5557 acc_corrupt_t_0p2_0p4=0.1392 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.2552 corrupt_frac_t_0p4_0p6=0.0762 acc_corrupt_t_0p6_0p8=0.3485 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=2.8612 out_g_norm=1.7761 acc_corrupt_t_0p8_1p0=0.4243 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.6891 init_gold_top10=0.2090 init_gold_top100=0.4276 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1378 init_acc_rollout_kept=0.1215 logit_acc_rollout_applied=0.1143 logit_acc_rollout_kept=0.1146 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=24.0s lr=2.000000e-03 loss=6.4546 loss_recon=6.4546 loss_meanflow=0.0000 mean_model_t=0.2098 mean_corrupt_t=0.2098 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5023 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1137 corrupt_frac=1.0000 acc_corrupt=0.1137 loss_corrupt=6.4546 wrong_frac=0.7902 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.0592 corrupt_frac_t_0p0_0p2=0.5544 acc_corrupt_t_0p2_0p4=0.1548 corrupt_frac_t_0p2_0p4=0.3617 acc_corrupt_t_0p4_0p6=0.2839 corrupt_frac_t_0p4_0p6=0.0743 out_w_norm=4.3408 out_g_norm=1.3199 acc_corrupt_t_0p6_0p8=0.3901 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.5415 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.2279 init_gold_top10=0.2020 init_gold_top100=0.4348 rollout_applied_pos_frac=0.4375 init_acc_rollout_applied=0.0878 init_acc_rollout_kept=0.1243 logit_acc_rollout_applied=0.1032 logit_acc_rollout_kept=0.1241 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=23.9s lr=2.000000e-03 loss=5.9837 loss_recon=5.9837 loss_meanflow=0.0000 mean_model_t=0.2072 mean_corrupt_t=0.2072 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1233 corrupt_frac=1.0000 acc_corrupt=0.1233 loss_corrupt=5.9837 wrong_frac=0.7929 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.0639 corrupt_frac_t_0p0_0p2=0.5638 acc_corrupt_t_0p2_0p4=0.1708 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.3106 corrupt_frac_t_0p4_0p6=0.0753 out_w_norm=5.4789 out_g_norm=0.5031 acc_corrupt_t_0p6_0p8=0.4367 corrupt_frac_t_0p6_0p8=0.0128 acc_corrupt_t_0p8_1p0=0.5306 corrupt_frac_t_0p8_1p0=0.0117 loss_all=5.7599 init_gold_top10=0.2017 init_gold_top100=0.4699 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1182 init_acc_rollout_kept=0.1042 logit_acc_rollout_applied=0.1310 logit_acc_rollout_kept=0.1227 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=23.9s lr=2.000000e-03 loss=5.4774 loss_recon=5.4774 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1378 corrupt_frac=1.0000 acc_corrupt=0.1378 loss_corrupt=5.4774 wrong_frac=0.7898 init_acc_corrupt=0.1197 acc_corrupt_t_0p0_0p2=0.0677 corrupt_frac_t_0p0_0p2=0.5506 acc_corrupt_t_0p2_0p4=0.1918 corrupt_frac_t_0p2_0p4=0.3660 acc_corrupt_t_0p4_0p6=0.3501 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.4913 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=6.7180 out_g_norm=0.4070 acc_corrupt_t_0p8_1p0=0.6104 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.1003 init_gold_top10=0.2273 init_gold_top100=0.5161 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.1306 init_acc_rollout_kept=0.1486 logit_acc_rollout_applied=0.1487 logit_acc_rollout_kept=0.1648 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=23.9s lr=2.000000e-03 loss=4.8712 loss_recon=4.8712 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5037 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1521 corrupt_frac=1.0000 acc_corrupt=0.1521 loss_corrupt=4.8712 wrong_frac=0.7918 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.0719 corrupt_frac_t_0p0_0p2=0.5629 acc_corrupt_t_0p2_0p4=0.2147 corrupt_frac_t_0p2_0p4=0.3502 acc_corrupt_t_0p4_0p6=0.4001 corrupt_frac_t_0p4_0p6=0.0768 out_w_norm=7.9804 out_g_norm=0.4280 acc_corrupt_t_0p6_0p8=0.5625 corrupt_frac_t_0p6_0p8=0.0133 loss_all=4.5656 init_gold_top10=0.2029 init_gold_top100=0.5925 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.1124 init_acc_rollout_kept=0.1078 logit_acc_rollout_applied=0.1600 logit_acc_rollout_kept=0.1533 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=24.1s lr=2.000000e-03 loss=4.2343 loss_recon=4.2343 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5123 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1770 corrupt_frac=1.0000 acc_corrupt=0.1770 loss_corrupt=4.2343 wrong_frac=0.7915 init_acc_corrupt=0.1192 acc_corrupt_t_0p0_0p2=0.0787 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.2531 corrupt_frac_t_0p2_0p4=0.3563 acc_corrupt_t_0p4_0p6=0.4880 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=9.1564 out_g_norm=0.4726 acc_corrupt_t_0p6_0p8=0.6622 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=0.8376 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.9427 init_gold_top10=0.2206 init_gold_top100=0.6230 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1283 init_acc_rollout_kept=0.1214 logit_acc_rollout_applied=0.1920 logit_acc_rollout_kept=0.1924 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=23.9s lr=2.000000e-03 loss=3.7296 loss_recon=3.7296 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5020 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2008 corrupt_frac=1.0000 acc_corrupt=0.2008 loss_corrupt=3.7296 wrong_frac=0.7911 init_acc_corrupt=0.1208 acc_corrupt_t_0p0_0p2=0.0884 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=0.2944 corrupt_frac_t_0p2_0p4=0.3674 acc_corrupt_t_0p4_0p6=0.5237 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.6942 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=9.9997 out_g_norm=0.5989 acc_corrupt_t_0p8_1p0=0.8555 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.4499 init_gold_top10=0.2440 init_gold_top100=0.5899 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.1209 init_acc_rollout_kept=0.1210 logit_acc_rollout_applied=0.2120 logit_acc_rollout_kept=0.2282 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=24.0s lr=2.000000e-03 loss=3.2983 loss_recon=3.2983 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5052 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2379 corrupt_frac=1.0000 acc_corrupt=0.2379 loss_corrupt=3.2983 wrong_frac=0.7905 init_acc_corrupt=0.1230 acc_corrupt_t_0p0_0p2=0.1028 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.3571 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.5909 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.7411 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=10.4993 out_g_norm=0.8729 acc_corrupt_t_0p8_1p0=0.8477 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.0155 init_gold_top10=0.2915 init_gold_top100=0.6315 rollout_applied_pos_frac=0.4766 init_acc_rollout_applied=0.1272 init_acc_rollout_kept=0.1170 logit_acc_rollout_applied=0.2742 logit_acc_rollout_kept=0.2656 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=23.8s lr=2.000000e-03 loss=2.8778 loss_recon=2.8778 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4952 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2973 corrupt_frac=1.0000 acc_corrupt=0.2973 loss_corrupt=2.8778 wrong_frac=0.7924 init_acc_corrupt=0.1228 acc_corrupt_t_0p0_0p2=0.1297 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=0.4607 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.6989 corrupt_frac_t_0p4_0p6=0.0751 acc_corrupt_t_0p6_0p8=0.8142 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=10.8390 out_g_norm=1.0522 loss_all=2.7461 init_gold_top10=0.3628 init_gold_top100=0.6665 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1368 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.3638 logit_acc_rollout_kept=0.2733 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=24.6s lr=2.000000e-03 loss=2.4561 loss_recon=2.4561 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3788 corrupt_frac=1.0000 acc_corrupt=0.3788 loss_corrupt=2.4561 wrong_frac=0.7915 init_acc_corrupt=0.1281 acc_corrupt_t_0p0_0p2=0.1728 corrupt_frac_t_0p0_0p2=0.5640 acc_corrupt_t_0p2_0p4=0.6004 corrupt_frac_t_0p2_0p4=0.3466 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p6_0p8=0.8828 corrupt_frac_t_0p6_0p8=0.0136 out_w_norm=11.1283 out_g_norm=1.2622 acc_corrupt_t_0p8_1p0=0.9307 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.1052 init_gold_top10=0.4033 init_gold_top100=0.6222 rollout_applied_pos_frac=0.4453 init_acc_rollout_applied=0.1418 init_acc_rollout_kept=0.1192 logit_acc_rollout_applied=0.4907 logit_acc_rollout_kept=0.4183 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=23.7s lr=2.000000e-03 loss=2.0694 loss_recon=2.0694 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4995 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4665 corrupt_frac=1.0000 acc_corrupt=0.4665 loss_corrupt=2.0694 wrong_frac=0.7905 init_acc_corrupt=0.1362 acc_corrupt_t_0p0_0p2=0.2250 corrupt_frac_t_0p0_0p2=0.5557 acc_corrupt_t_0p2_0p4=0.7365 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9014 corrupt_frac_t_0p4_0p6=0.0762 acc_corrupt_t_0p6_0p8=0.9335 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.3914 out_g_norm=1.4567 acc_corrupt_t_0p8_1p0=0.9663 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.9323 init_gold_top10=0.4233 init_gold_top100=0.6392 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1810 init_acc_rollout_kept=0.1215 logit_acc_rollout_applied=0.5332 logit_acc_rollout_kept=0.4872 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=23.7s lr=2.000000e-03 loss=1.7590 loss_recon=1.7590 loss_meanflow=0.0000 mean_model_t=0.2098 mean_corrupt_t=0.2098 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5023 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5421 corrupt_frac=1.0000 acc_corrupt=0.5421 loss_corrupt=1.7590 wrong_frac=0.7902 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.2864 corrupt_frac_t_0p0_0p2=0.5544 acc_corrupt_t_0p2_0p4=0.8375 corrupt_frac_t_0p2_0p4=0.3617 acc_corrupt_t_0p4_0p6=0.9568 corrupt_frac_t_0p4_0p6=0.0743 out_w_norm=11.5756 out_g_norm=1.5131 acc_corrupt_t_0p6_0p8=0.9688 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.9624 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4657 init_gold_top10=0.4631 init_gold_top100=0.6198 rollout_applied_pos_frac=0.4375 init_acc_rollout_applied=0.1669 init_acc_rollout_kept=0.1243 logit_acc_rollout_applied=0.6838 logit_acc_rollout_kept=0.5458 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=23.7s lr=2.000000e-03 loss=1.5170 loss_recon=1.5170 loss_meanflow=0.0000 mean_model_t=0.2072 mean_corrupt_t=0.2072 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6018 corrupt_frac=1.0000 acc_corrupt=0.6018 loss_corrupt=1.5170 wrong_frac=0.7929 init_acc_corrupt=0.1570 acc_corrupt_t_0p0_0p2=0.3584 corrupt_frac_t_0p0_0p2=0.5638 acc_corrupt_t_0p2_0p4=0.9009 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.9811 corrupt_frac_t_0p4_0p6=0.0753 out_w_norm=11.6982 out_g_norm=1.5191 acc_corrupt_t_0p6_0p8=0.9840 corrupt_frac_t_0p6_0p8=0.0128 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0117 loss_all=1.3672 init_gold_top10=0.4951 init_gold_top100=0.6454 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.2290 init_acc_rollout_kept=0.1042 logit_acc_rollout_applied=0.7189 logit_acc_rollout_kept=0.5401 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=23.7s lr=2.000000e-03 loss=1.2915 loss_recon=1.2915 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6540 corrupt_frac=1.0000 acc_corrupt=0.6540 loss_corrupt=1.2915 wrong_frac=0.7898 init_acc_corrupt=0.1734 acc_corrupt_t_0p0_0p2=0.4134 corrupt_frac_t_0p0_0p2=0.5506 acc_corrupt_t_0p2_0p4=0.9392 corrupt_frac_t_0p2_0p4=0.3660 acc_corrupt_t_0p4_0p6=0.9911 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9905 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7810 out_g_norm=1.4652 acc_corrupt_t_0p8_1p0=0.9917 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0025 init_gold_top10=0.5435 init_gold_top100=0.6617 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2334 init_acc_rollout_kept=0.1486 logit_acc_rollout_applied=0.7939 logit_acc_rollout_kept=0.6810 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=23.7s lr=2.000000e-03 loss=1.1774 loss_recon=1.1774 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5037 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6826 corrupt_frac=1.0000 acc_corrupt=0.6826 loss_corrupt=1.1774 wrong_frac=0.7918 init_acc_corrupt=0.1812 acc_corrupt_t_0p0_0p2=0.4607 corrupt_frac_t_0p0_0p2=0.5629 acc_corrupt_t_0p2_0p4=0.9616 corrupt_frac_t_0p2_0p4=0.3502 acc_corrupt_t_0p4_0p6=0.9956 corrupt_frac_t_0p4_0p6=0.0768 out_w_norm=11.8241 out_g_norm=1.3619 acc_corrupt_t_0p6_0p8=0.9946 corrupt_frac_t_0p6_0p8=0.0133 loss_all=1.1961 init_gold_top10=0.5235 init_gold_top100=0.6535 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2526 init_acc_rollout_kept=0.1078 logit_acc_rollout_applied=0.8093 logit_acc_rollout_kept=0.5727 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=23.8s lr=2.000000e-03 loss=1.0426 loss_recon=1.0426 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5123 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7140 corrupt_frac=1.0000 acc_corrupt=0.7140 loss_corrupt=1.0426 wrong_frac=0.7915 init_acc_corrupt=0.1897 acc_corrupt_t_0p0_0p2=0.5042 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.9759 corrupt_frac_t_0p2_0p4=0.3563 acc_corrupt_t_0p4_0p6=0.9978 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=11.8438 out_g_norm=1.2704 acc_corrupt_t_0p6_0p8=0.9969 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0589 init_gold_top10=0.5409 init_gold_top100=0.6740 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.2758 init_acc_rollout_kept=0.1214 logit_acc_rollout_applied=0.7361 logit_acc_rollout_kept=0.6505 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=23.7s lr=2.000000e-03 loss=0.9326 loss_recon=0.9326 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5020 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7394 corrupt_frac=1.0000 acc_corrupt=0.7394 loss_corrupt=0.9326 wrong_frac=0.7911 init_acc_corrupt=0.1967 acc_corrupt_t_0p0_0p2=0.5387 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=0.9829 corrupt_frac_t_0p2_0p4=0.3674 acc_corrupt_t_0p4_0p6=0.9988 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9967 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=11.8643 out_g_norm=1.1264 acc_corrupt_t_0p8_1p0=0.9866 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7271 init_gold_top10=0.5306 init_gold_top100=0.6286 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.2711 init_acc_rollout_kept=0.1210 logit_acc_rollout_applied=0.8536 logit_acc_rollout_kept=0.7328 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=23.8s lr=2.000000e-03 loss=0.8706 loss_recon=0.8706 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5052 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7491 corrupt_frac=1.0000 acc_corrupt=0.7491 loss_corrupt=0.8706 wrong_frac=0.7905 init_acc_corrupt=0.2009 acc_corrupt_t_0p0_0p2=0.5529 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9885 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.9988 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9971 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.8558 out_g_norm=1.0997 acc_corrupt_t_0p8_1p0=0.9964 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7654 init_gold_top10=0.5507 init_gold_top100=0.6459 rollout_applied_pos_frac=0.4766 init_acc_rollout_applied=0.2787 init_acc_rollout_kept=0.1170 logit_acc_rollout_applied=0.8564 logit_acc_rollout_kept=0.7075 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=23.7s lr=2.000000e-03 loss=0.7684 loss_recon=0.7684 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4952 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7720 corrupt_frac=1.0000 acc_corrupt=0.7720 loss_corrupt=0.7684 wrong_frac=0.7924 init_acc_corrupt=0.1993 acc_corrupt_t_0p0_0p2=0.5971 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=0.9900 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.9990 corrupt_frac_t_0p4_0p6=0.0751 acc_corrupt_t_0p6_0p8=0.9969 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8523 out_g_norm=1.1152 loss_all=0.7562 init_gold_top10=0.5871 init_gold_top100=0.6723 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.3057 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.8930 logit_acc_rollout_kept=0.6352 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p50_path4_unif0_0p125_outwdm1_ctx1024_sampledpath_true_20260517_224139/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=24.6s lr=2.000000e-03 loss=0.6727 loss_recon=0.6727 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7942 corrupt_frac=1.0000 acc_corrupt=0.7942 loss_corrupt=0.6727 wrong_frac=0.7915 init_acc_corrupt=0.2065 acc_corrupt_t_0p0_0p2=0.6391 corrupt_frac_t_0p0_0p2=0.5640 acc_corrupt_t_0p2_0p4=0.9938 corrupt_frac_t_0p2_0p4=0.3466 acc_corrupt_t_0p4_0p6=0.9993 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p6_0p8=0.9977 corrupt_frac_t_0p6_0p8=0.0136 out_w_norm=11.8640 out_g_norm=1.0068 acc_corrupt_t_0p8_1p0=0.9895 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5473 init_gold_top10=0.5368 init_gold_top100=0.6265 rollout_applied_pos_frac=0.4453 init_acc_rollout_applied=0.3061 init_acc_rollout_kept=0.1192 logit_acc_rollout_applied=0.8278 logit_acc_rollout_kept=0.8275 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=23.6s lr=2.000000e-03 loss=0.5787 loss_recon=0.5787 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4995 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8214 corrupt_frac=1.0000 acc_corrupt=0.8214 loss_corrupt=0.5787 wrong_frac=0.7905 init_acc_corrupt=0.2095 acc_corrupt_t_0p0_0p2=0.6810 corrupt_frac_t_0p0_0p2=0.5557 acc_corrupt_t_0p2_0p4=0.9966 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.0762 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8582 out_g_norm=0.9122 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5341 init_gold_top10=0.5514 init_gold_top100=0.6421 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.2945 init_acc_rollout_kept=0.1215 logit_acc_rollout_applied=0.8562 logit_acc_rollout_kept=0.8293 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705.log new file mode 100644 index 0000000000000000000000000000000000000000..8f94f37e5cdfadb750ee7b0686ac472345030989 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705.log @@ -0,0 +1,1433 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=30.6s lr=2.000000e-03 loss=7.7217 loss_recon=7.7217 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0950 corrupt_frac=1.0000 acc_corrupt=0.0950 loss_corrupt=7.7217 wrong_frac=0.7918 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.0500 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1258 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2490 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3463 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0047 out_g_norm=1.0904 loss_all=7.4661 init_gold_top10=0.2043 init_gold_top100=0.4569 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1259 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.1050 logit_acc_rollout_kept=0.0858 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=29.5s lr=2.000000e-03 loss=7.0822 loss_recon=7.0822 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1045 corrupt_frac=1.0000 acc_corrupt=0.1045 loss_corrupt=7.0822 wrong_frac=0.7893 init_acc_corrupt=0.1183 acc_corrupt_t_0p0_0p2=0.0563 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1400 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2541 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.3502 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=2.8691 out_g_norm=1.7713 acc_corrupt_t_0p8_1p0=0.4316 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.7577 init_gold_top10=0.1946 init_gold_top100=0.4900 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.0942 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.0979 logit_acc_rollout_kept=0.1220 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=29.7s lr=2.000000e-03 loss=6.4685 loss_recon=6.4685 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1122 corrupt_frac=1.0000 acc_corrupt=0.1122 loss_corrupt=6.4685 wrong_frac=0.7934 init_acc_corrupt=0.1153 acc_corrupt_t_0p0_0p2=0.0591 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1550 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.2818 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.3966 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=4.3679 out_g_norm=1.3331 acc_corrupt_t_0p8_1p0=0.5368 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.2025 init_gold_top10=0.2021 init_gold_top100=0.5078 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1151 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.1140 logit_acc_rollout_kept=0.1313 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=29.7s lr=2.000000e-03 loss=5.9858 loss_recon=5.9858 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1247 corrupt_frac=1.0000 acc_corrupt=0.1247 loss_corrupt=5.9858 wrong_frac=0.7914 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.0636 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.1722 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.3163 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=5.4952 out_g_norm=0.5217 acc_corrupt_t_0p6_0p8=0.4375 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.5708 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8211 init_gold_top10=0.1957 init_gold_top100=0.5286 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.0842 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.1087 logit_acc_rollout_kept=0.1548 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=29.8s lr=2.000000e-03 loss=5.5038 loss_recon=5.5038 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1360 corrupt_frac=1.0000 acc_corrupt=0.1360 loss_corrupt=5.5038 wrong_frac=0.7930 init_acc_corrupt=0.1166 acc_corrupt_t_0p0_0p2=0.0677 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.1906 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.3551 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.4913 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.5674 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=6.7373 out_g_norm=0.3896 loss_all=5.1370 init_gold_top10=0.2115 init_gold_top100=0.5998 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1262 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.1488 logit_acc_rollout_kept=0.1512 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=29.7s lr=2.000000e-03 loss=4.8739 loss_recon=4.8739 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1531 corrupt_frac=1.0000 acc_corrupt=0.1531 loss_corrupt=4.8739 wrong_frac=0.7925 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.0735 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.2173 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.4039 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.5645 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=8.0308 out_g_norm=0.4103 acc_corrupt_t_0p8_1p0=0.6416 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.6775 init_gold_top10=0.1974 init_gold_top100=0.6800 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1035 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.1467 logit_acc_rollout_kept=0.1668 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=29.6s lr=2.000000e-03 loss=4.2328 loss_recon=4.2328 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1803 corrupt_frac=1.0000 acc_corrupt=0.1803 loss_corrupt=4.2328 wrong_frac=0.7907 init_acc_corrupt=0.1213 acc_corrupt_t_0p0_0p2=0.0791 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.2578 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.4915 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.6661 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.2239 out_g_norm=0.5138 acc_corrupt_t_0p8_1p0=0.8584 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.9215 init_gold_top10=0.2364 init_gold_top100=0.7608 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1299 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.1972 logit_acc_rollout_kept=0.2063 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=29.7s lr=2.000000e-03 loss=3.7333 loss_recon=3.7333 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2059 corrupt_frac=1.0000 acc_corrupt=0.2059 loss_corrupt=3.7333 wrong_frac=0.7901 init_acc_corrupt=0.1232 acc_corrupt_t_0p0_0p2=0.0881 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.3034 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.5421 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=10.0940 out_g_norm=0.6406 acc_corrupt_t_0p6_0p8=0.7009 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.8389 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.3739 init_gold_top10=0.2771 init_gold_top100=0.8236 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.1434 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.2439 logit_acc_rollout_kept=0.2537 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=29.6s lr=2.000000e-03 loss=3.2681 loss_recon=3.2681 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2475 corrupt_frac=1.0000 acc_corrupt=0.2475 loss_corrupt=3.2681 wrong_frac=0.7895 init_acc_corrupt=0.1257 acc_corrupt_t_0p0_0p2=0.1045 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.3727 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.6185 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.7589 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=10.6429 out_g_norm=0.7657 acc_corrupt_t_0p8_1p0=0.8522 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.0748 init_gold_top10=0.3044 init_gold_top100=0.7720 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1230 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.2735 logit_acc_rollout_kept=0.2504 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=29.6s lr=2.000000e-03 loss=2.8411 loss_recon=2.8411 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3098 corrupt_frac=1.0000 acc_corrupt=0.3098 loss_corrupt=2.8411 wrong_frac=0.7888 init_acc_corrupt=0.1283 acc_corrupt_t_0p0_0p2=0.1305 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.4787 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.7201 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=11.0232 out_g_norm=0.9740 acc_corrupt_t_0p6_0p8=0.8239 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9121 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.5752 init_gold_top10=0.4164 init_gold_top100=0.8336 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1345 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.3515 logit_acc_rollout_kept=0.3856 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=30.3s lr=2.000000e-03 loss=2.4282 loss_recon=2.4282 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3888 corrupt_frac=1.0000 acc_corrupt=0.3888 loss_corrupt=2.4282 wrong_frac=0.7918 init_acc_corrupt=0.1269 acc_corrupt_t_0p0_0p2=0.1729 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.6175 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.8282 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.8874 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.3189 out_g_norm=1.0872 loss_all=2.0140 init_gold_top10=0.5052 init_gold_top100=0.7876 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1478 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.5084 logit_acc_rollout_kept=0.3695 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=29.4s lr=2.000000e-03 loss=2.0460 loss_recon=2.0460 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4754 corrupt_frac=1.0000 acc_corrupt=0.4754 loss_corrupt=2.0460 wrong_frac=0.7893 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.2289 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.7469 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9075 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9414 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5680 out_g_norm=1.2941 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.8759 init_gold_top10=0.5238 init_gold_top100=0.8236 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1149 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.5107 logit_acc_rollout_kept=0.5327 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=29.4s lr=2.000000e-03 loss=1.8078 loss_recon=1.8078 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5360 corrupt_frac=1.0000 acc_corrupt=0.5360 loss_corrupt=1.8078 wrong_frac=0.7934 init_acc_corrupt=0.1319 acc_corrupt_t_0p0_0p2=0.2793 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.8472 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9585 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9702 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.7558 out_g_norm=1.3538 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.0081 init_gold_top10=0.5289 init_gold_top100=0.8086 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1365 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.5554 logit_acc_rollout_kept=0.4754 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=29.5s lr=2.000000e-03 loss=1.5540 loss_recon=1.5540 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5961 corrupt_frac=1.0000 acc_corrupt=0.5961 loss_corrupt=1.5540 wrong_frac=0.7914 init_acc_corrupt=0.1382 acc_corrupt_t_0p0_0p2=0.3363 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9088 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9798 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=11.8876 out_g_norm=1.3224 acc_corrupt_t_0p6_0p8=0.9842 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.5301 init_gold_top10=0.5708 init_gold_top100=0.8118 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1124 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.6175 logit_acc_rollout_kept=0.5716 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=29.6s lr=2.000000e-03 loss=1.3485 loss_recon=1.3485 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6414 corrupt_frac=1.0000 acc_corrupt=0.6414 loss_corrupt=1.3485 wrong_frac=0.7930 init_acc_corrupt=0.1400 acc_corrupt_t_0p0_0p2=0.4028 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9398 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9884 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9887 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.9532 out_g_norm=1.2315 loss_all=1.2878 init_gold_top10=0.6049 init_gold_top100=0.8215 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1609 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.6847 logit_acc_rollout_kept=0.5399 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=29.5s lr=2.000000e-03 loss=1.1365 loss_recon=1.1365 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6899 corrupt_frac=1.0000 acc_corrupt=0.6899 loss_corrupt=1.1365 wrong_frac=0.7925 init_acc_corrupt=0.1436 acc_corrupt_t_0p0_0p2=0.4736 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9612 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9934 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9929 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.0072 out_g_norm=1.1381 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2448 init_gold_top10=0.6246 init_gold_top100=0.8349 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1379 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.6815 logit_acc_rollout_kept=0.5510 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=29.4s lr=2.000000e-03 loss=0.9634 loss_recon=0.9634 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7302 corrupt_frac=1.0000 acc_corrupt=0.7302 loss_corrupt=0.9634 wrong_frac=0.7907 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.5331 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9714 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9958 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9944 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.0606 out_g_norm=1.0952 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9649 init_gold_top10=0.6894 init_gold_top100=0.8433 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1662 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.7649 logit_acc_rollout_kept=0.6903 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=29.5s lr=2.000000e-03 loss=0.7820 loss_recon=0.7820 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7733 corrupt_frac=1.0000 acc_corrupt=0.7733 loss_corrupt=0.7820 wrong_frac=0.7901 init_acc_corrupt=0.1538 acc_corrupt_t_0p0_0p2=0.6021 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9842 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9972 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.0975 out_g_norm=0.9588 acc_corrupt_t_0p6_0p8=0.9949 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6225 init_gold_top10=0.7797 init_gold_top100=0.8787 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.1888 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.8382 logit_acc_rollout_kept=0.7002 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=29.4s lr=2.000000e-03 loss=0.6553 loss_recon=0.6553 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8014 corrupt_frac=1.0000 acc_corrupt=0.8014 loss_corrupt=0.6553 wrong_frac=0.7895 init_acc_corrupt=0.1574 acc_corrupt_t_0p0_0p2=0.6458 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9920 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9981 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9965 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.1295 out_g_norm=1.0121 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6492 init_gold_top10=0.7015 init_gold_top100=0.8028 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1702 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8124 logit_acc_rollout_kept=0.7423 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=29.4s lr=2.000000e-03 loss=0.5856 loss_recon=0.5856 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8161 corrupt_frac=1.0000 acc_corrupt=0.8161 loss_corrupt=0.5856 wrong_frac=0.7888 init_acc_corrupt=0.1603 acc_corrupt_t_0p0_0p2=0.6696 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9957 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9988 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.1168 out_g_norm=0.9118 acc_corrupt_t_0p6_0p8=0.9973 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9945 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5609 init_gold_top10=0.7788 init_gold_top100=0.8592 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1737 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.8161 logit_acc_rollout_kept=0.8383 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=30.4s lr=2.000000e-03 loss=0.5554 loss_recon=0.5554 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8234 corrupt_frac=1.0000 acc_corrupt=0.8234 loss_corrupt=0.5554 wrong_frac=0.7918 init_acc_corrupt=0.1574 acc_corrupt_t_0p0_0p2=0.6840 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9976 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9992 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.0753 out_g_norm=0.8153 loss_all=0.3933 init_gold_top10=0.7284 init_gold_top100=0.8007 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1919 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.9182 logit_acc_rollout_kept=0.8265 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=29.4s lr=2.000000e-03 loss=0.5081 loss_recon=0.5081 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8361 corrupt_frac=1.0000 acc_corrupt=0.8361 loss_corrupt=0.5081 wrong_frac=0.7893 init_acc_corrupt=0.1603 acc_corrupt_t_0p0_0p2=0.7044 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9993 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9983 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.0229 out_g_norm=0.7929 acc_corrupt_t_0p8_1p0=0.9967 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5020 init_gold_top10=0.7600 init_gold_top100=0.8294 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1499 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.8341 logit_acc_rollout_kept=0.8102 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=29.4s lr=2.000000e-03 loss=0.5018 loss_recon=0.5018 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8376 corrupt_frac=1.0000 acc_corrupt=0.8376 loss_corrupt=0.5018 wrong_frac=0.7934 init_acc_corrupt=0.1566 acc_corrupt_t_0p0_0p2=0.7129 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9988 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9986 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.9897 out_g_norm=0.6785 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6487 init_gold_top10=0.7186 init_gold_top100=0.8177 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1618 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.8117 logit_acc_rollout_kept=0.7786 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=29.5s lr=2.000000e-03 loss=0.4696 loss_recon=0.4696 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8484 corrupt_frac=1.0000 acc_corrupt=0.8484 loss_corrupt=0.4696 wrong_frac=0.7914 init_acc_corrupt=0.1595 acc_corrupt_t_0p0_0p2=0.7283 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=11.9609 out_g_norm=0.6910 acc_corrupt_t_0p6_0p8=0.9986 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9927 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5296 init_gold_top10=0.7328 init_gold_top100=0.8143 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1386 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.8328 logit_acc_rollout_kept=0.8467 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=29.6s lr=2.000000e-03 loss=0.4438 loss_recon=0.4438 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8559 corrupt_frac=1.0000 acc_corrupt=0.8559 loss_corrupt=0.4438 wrong_frac=0.7930 init_acc_corrupt=0.1582 acc_corrupt_t_0p0_0p2=0.7446 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9988 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.9292 out_g_norm=0.6275 loss_all=0.4529 init_gold_top10=0.7434 init_gold_top100=0.8219 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1820 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.8422 logit_acc_rollout_kept=0.8593 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=29.5s lr=2.000000e-03 loss=0.4126 loss_recon=0.4126 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8655 corrupt_frac=1.0000 acc_corrupt=0.8655 loss_corrupt=0.4126 wrong_frac=0.7925 init_acc_corrupt=0.1589 acc_corrupt_t_0p0_0p2=0.7609 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.9169 out_g_norm=0.5527 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5004 init_gold_top10=0.7639 init_gold_top100=0.8351 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1555 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.8555 logit_acc_rollout_kept=0.8082 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=29.4s lr=2.000000e-03 loss=0.3924 loss_recon=0.3924 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8728 corrupt_frac=1.0000 acc_corrupt=0.8728 loss_corrupt=0.3924 wrong_frac=0.7907 init_acc_corrupt=0.1620 acc_corrupt_t_0p0_0p2=0.7710 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8976 out_g_norm=0.4890 acc_corrupt_t_0p8_1p0=0.9971 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4354 init_gold_top10=0.7814 init_gold_top100=0.8433 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1809 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.8855 logit_acc_rollout_kept=0.8572 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=29.5s lr=2.000000e-03 loss=0.3808 loss_recon=0.3808 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8777 corrupt_frac=1.0000 acc_corrupt=0.8777 loss_corrupt=0.3808 wrong_frac=0.7901 init_acc_corrupt=0.1628 acc_corrupt_t_0p0_0p2=0.7798 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=11.8812 out_g_norm=0.4824 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9971 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4072 init_gold_top10=0.8211 init_gold_top100=0.8787 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.2021 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.8815 logit_acc_rollout_kept=0.8569 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=29.4s lr=2.000000e-03 loss=0.3527 loss_recon=0.3527 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8852 corrupt_frac=1.0000 acc_corrupt=0.8852 loss_corrupt=0.3527 wrong_frac=0.7895 init_acc_corrupt=0.1639 acc_corrupt_t_0p0_0p2=0.7923 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.8910 out_g_norm=0.4698 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4169 init_gold_top10=0.7383 init_gold_top100=0.8028 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1767 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8819 logit_acc_rollout_kept=0.8343 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=29.4s lr=2.000000e-03 loss=0.3332 loss_recon=0.3332 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8924 corrupt_frac=1.0000 acc_corrupt=0.8924 loss_corrupt=0.3332 wrong_frac=0.7888 init_acc_corrupt=0.1657 acc_corrupt_t_0p0_0p2=0.8052 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=11.9066 out_g_norm=0.4173 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9969 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3573 init_gold_top10=0.8069 init_gold_top100=0.8592 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1787 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.8856 logit_acc_rollout_kept=0.9172 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=30.3s lr=2.000000e-03 loss=0.3352 loss_recon=0.3352 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8915 corrupt_frac=1.0000 acc_corrupt=0.8915 loss_corrupt=0.3352 wrong_frac=0.7918 init_acc_corrupt=0.1619 acc_corrupt_t_0p0_0p2=0.8050 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.9193 out_g_norm=0.4141 loss_all=0.1708 init_gold_top10=0.7507 init_gold_top100=0.8007 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1986 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.9597 logit_acc_rollout_kept=0.9032 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=29.3s lr=2.000000e-03 loss=0.3085 loss_recon=0.3085 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9009 corrupt_frac=1.0000 acc_corrupt=0.9009 loss_corrupt=0.3085 wrong_frac=0.7893 init_acc_corrupt=0.1648 acc_corrupt_t_0p0_0p2=0.8205 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.9391 out_g_norm=0.3847 acc_corrupt_t_0p8_1p0=0.9977 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3074 init_gold_top10=0.7739 init_gold_top100=0.8294 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1558 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.9042 logit_acc_rollout_kept=0.8959 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=29.4s lr=2.000000e-03 loss=0.3175 loss_recon=0.3175 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8985 corrupt_frac=1.0000 acc_corrupt=0.8985 loss_corrupt=0.3175 wrong_frac=0.7934 init_acc_corrupt=0.1604 acc_corrupt_t_0p0_0p2=0.8202 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.9604 out_g_norm=0.4031 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4360 init_gold_top10=0.7488 init_gold_top100=0.8177 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1671 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.8646 logit_acc_rollout_kept=0.9160 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=29.5s lr=2.000000e-03 loss=0.3043 loss_recon=0.3043 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9028 corrupt_frac=1.0000 acc_corrupt=0.9028 loss_corrupt=0.3043 wrong_frac=0.7914 init_acc_corrupt=0.1630 acc_corrupt_t_0p0_0p2=0.8254 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=11.9889 out_g_norm=0.3439 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9966 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4685 init_gold_top10=0.7513 init_gold_top100=0.8143 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1409 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.8589 logit_acc_rollout_kept=0.8975 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=29.6s lr=2.000000e-03 loss=0.2964 loss_recon=0.2964 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9052 corrupt_frac=1.0000 acc_corrupt=0.9052 loss_corrupt=0.2964 wrong_frac=0.7930 init_acc_corrupt=0.1617 acc_corrupt_t_0p0_0p2=0.8317 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.0195 out_g_norm=0.3291 loss_all=0.3202 init_gold_top10=0.7630 init_gold_top100=0.8219 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1872 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.8809 logit_acc_rollout_kept=0.9150 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=29.5s lr=2.000000e-03 loss=0.2894 loss_recon=0.2894 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9076 corrupt_frac=1.0000 acc_corrupt=0.9076 loss_corrupt=0.2894 wrong_frac=0.7925 init_acc_corrupt=0.1618 acc_corrupt_t_0p0_0p2=0.8356 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.0470 out_g_norm=0.3113 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3613 init_gold_top10=0.7700 init_gold_top100=0.8351 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1580 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.8884 logit_acc_rollout_kept=0.8595 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=29.4s lr=2.000000e-03 loss=0.2858 loss_recon=0.2858 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9075 corrupt_frac=1.0000 acc_corrupt=0.9075 loss_corrupt=0.2858 wrong_frac=0.7907 init_acc_corrupt=0.1646 acc_corrupt_t_0p0_0p2=0.8334 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.0751 out_g_norm=0.2856 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3303 init_gold_top10=0.7881 init_gold_top100=0.8433 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1848 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.8976 logit_acc_rollout_kept=0.9435 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=29.5s lr=2.000000e-03 loss=0.2803 loss_recon=0.2803 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9111 corrupt_frac=1.0000 acc_corrupt=0.9111 loss_corrupt=0.2803 wrong_frac=0.7901 init_acc_corrupt=0.1654 acc_corrupt_t_0p0_0p2=0.8399 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.1005 out_g_norm=0.2821 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2498 init_gold_top10=0.8342 init_gold_top100=0.8787 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.2042 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.9342 logit_acc_rollout_kept=0.9128 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=29.4s lr=2.000000e-03 loss=0.2775 loss_recon=0.2775 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9114 corrupt_frac=1.0000 acc_corrupt=0.9114 loss_corrupt=0.2775 wrong_frac=0.7895 init_acc_corrupt=0.1660 acc_corrupt_t_0p0_0p2=0.8397 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.1370 out_g_norm=0.2775 acc_corrupt_t_0p8_1p0=0.9984 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3296 init_gold_top10=0.7425 init_gold_top100=0.8028 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1810 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8979 logit_acc_rollout_kept=0.8910 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=29.4s lr=2.000000e-03 loss=0.2768 loss_recon=0.2768 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9125 corrupt_frac=1.0000 acc_corrupt=0.9125 loss_corrupt=0.2768 wrong_frac=0.7888 init_acc_corrupt=0.1674 acc_corrupt_t_0p0_0p2=0.8414 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.1788 out_g_norm=0.2558 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9977 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3145 init_gold_top10=0.8122 init_gold_top100=0.8592 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1810 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.9027 logit_acc_rollout_kept=0.9147 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=30.3s lr=2.000000e-03 loss=0.2828 loss_recon=0.2828 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9105 corrupt_frac=1.0000 acc_corrupt=0.9105 loss_corrupt=0.2828 wrong_frac=0.7918 init_acc_corrupt=0.1634 acc_corrupt_t_0p0_0p2=0.8391 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.2001 out_g_norm=0.2622 loss_all=0.1281 init_gold_top10=0.7549 init_gold_top100=0.8007 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.2012 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.9688 logit_acc_rollout_kept=0.9308 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=29.3s lr=2.000000e-03 loss=0.2543 loss_recon=0.2543 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9194 corrupt_frac=1.0000 acc_corrupt=0.9194 loss_corrupt=0.2543 wrong_frac=0.7893 init_acc_corrupt=0.1660 acc_corrupt_t_0p0_0p2=0.8539 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.2327 out_g_norm=0.2590 acc_corrupt_t_0p8_1p0=0.9977 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2295 init_gold_top10=0.7792 init_gold_top100=0.8294 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1565 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.9161 logit_acc_rollout_kept=0.9426 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=29.4s lr=2.000000e-03 loss=0.2689 loss_recon=0.2689 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9148 corrupt_frac=1.0000 acc_corrupt=0.9148 loss_corrupt=0.2689 wrong_frac=0.7934 init_acc_corrupt=0.1618 acc_corrupt_t_0p0_0p2=0.8491 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.2569 out_g_norm=0.2317 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3458 init_gold_top10=0.7536 init_gold_top100=0.8177 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1678 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.8737 logit_acc_rollout_kept=0.9208 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=29.5s lr=2.000000e-03 loss=0.2629 loss_recon=0.2629 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9169 corrupt_frac=1.0000 acc_corrupt=0.9169 loss_corrupt=0.2629 wrong_frac=0.7914 init_acc_corrupt=0.1642 acc_corrupt_t_0p0_0p2=0.8508 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.2704 out_g_norm=0.2224 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3491 init_gold_top10=0.7643 init_gold_top100=0.8143 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1432 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.8867 logit_acc_rollout_kept=0.9271 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=29.5s lr=2.000000e-03 loss=0.2710 loss_recon=0.2710 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9138 corrupt_frac=1.0000 acc_corrupt=0.9138 loss_corrupt=0.2710 wrong_frac=0.7930 init_acc_corrupt=0.1625 acc_corrupt_t_0p0_0p2=0.8471 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.2932 out_g_norm=0.2208 loss_all=0.3631 init_gold_top10=0.7597 init_gold_top100=0.8219 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1885 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.8665 logit_acc_rollout_kept=0.9402 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=29.5s lr=2.000000e-03 loss=0.2547 loss_recon=0.2547 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9199 corrupt_frac=1.0000 acc_corrupt=0.9199 loss_corrupt=0.2547 wrong_frac=0.7925 init_acc_corrupt=0.1625 acc_corrupt_t_0p0_0p2=0.8574 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.3190 out_g_norm=0.2045 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3685 init_gold_top10=0.7770 init_gold_top100=0.8351 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1587 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.9047 logit_acc_rollout_kept=0.8421 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=29.4s lr=2.000000e-03 loss=0.2589 loss_recon=0.2589 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9181 corrupt_frac=1.0000 acc_corrupt=0.9181 loss_corrupt=0.2589 wrong_frac=0.7907 init_acc_corrupt=0.1653 acc_corrupt_t_0p0_0p2=0.8524 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.3324 out_g_norm=0.2109 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1947 init_gold_top10=0.8036 init_gold_top100=0.8433 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1862 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.9492 logit_acc_rollout_kept=0.9316 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=29.5s lr=2.000000e-03 loss=0.2534 loss_recon=0.2534 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9210 corrupt_frac=1.0000 acc_corrupt=0.9210 loss_corrupt=0.2534 wrong_frac=0.7901 init_acc_corrupt=0.1661 acc_corrupt_t_0p0_0p2=0.8577 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.3434 out_g_norm=0.2032 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1892 init_gold_top10=0.8397 init_gold_top100=0.8787 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.2049 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.9343 logit_acc_rollout_kept=0.9777 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=29.4s lr=2.000000e-03 loss=0.2548 loss_recon=0.2548 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9193 corrupt_frac=1.0000 acc_corrupt=0.9193 loss_corrupt=0.2548 wrong_frac=0.7895 init_acc_corrupt=0.1666 acc_corrupt_t_0p0_0p2=0.8539 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.3665 out_g_norm=0.2000 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3126 init_gold_top10=0.7396 init_gold_top100=0.8028 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1809 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8975 logit_acc_rollout_kept=0.9297 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=29.4s lr=2.000000e-03 loss=0.2494 loss_recon=0.2494 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9209 corrupt_frac=1.0000 acc_corrupt=0.9209 loss_corrupt=0.2494 wrong_frac=0.7888 init_acc_corrupt=0.1679 acc_corrupt_t_0p0_0p2=0.8567 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.3906 out_g_norm=0.1864 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9910 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3069 init_gold_top10=0.7995 init_gold_top100=0.8592 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1821 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.8931 logit_acc_rollout_kept=0.9383 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=5100 epoch=5100/6000 epoch_step=1/1 micro_steps=5100 elapsed=30.3s lr=2.000000e-03 loss=0.2591 loss_recon=0.2591 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9173 corrupt_frac=1.0000 acc_corrupt=0.9173 loss_corrupt=0.2591 wrong_frac=0.7918 init_acc_corrupt=0.1639 acc_corrupt_t_0p0_0p2=0.8513 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.4042 out_g_norm=0.1904 loss_all=0.1250 init_gold_top10=0.7475 init_gold_top100=0.8007 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.2024 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.9675 logit_acc_rollout_kept=0.9615 +step=5200 epoch=5200/6000 epoch_step=1/1 micro_steps=5200 elapsed=29.4s lr=2.000000e-03 loss=0.2310 loss_recon=0.2310 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9262 corrupt_frac=1.0000 acc_corrupt=0.9262 loss_corrupt=0.2310 wrong_frac=0.7893 init_acc_corrupt=0.1666 acc_corrupt_t_0p0_0p2=0.8663 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.4212 out_g_norm=0.1756 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2837 init_gold_top10=0.7707 init_gold_top100=0.8294 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1579 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.9137 logit_acc_rollout_kept=0.9742 +step=5300 epoch=5300/6000 epoch_step=1/1 micro_steps=5300 elapsed=29.4s lr=2.000000e-03 loss=0.2452 loss_recon=0.2452 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9235 corrupt_frac=1.0000 acc_corrupt=0.9235 loss_corrupt=0.2452 wrong_frac=0.7934 init_acc_corrupt=0.1620 acc_corrupt_t_0p0_0p2=0.8644 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.4360 out_g_norm=0.1846 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2990 init_gold_top10=0.7536 init_gold_top100=0.8177 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1685 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.9182 logit_acc_rollout_kept=0.9290 +step=5400 epoch=5400/6000 epoch_step=1/1 micro_steps=5400 elapsed=29.5s lr=2.000000e-03 loss=0.2466 loss_recon=0.2466 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9222 corrupt_frac=1.0000 acc_corrupt=0.9222 loss_corrupt=0.2466 wrong_frac=0.7914 init_acc_corrupt=0.1644 acc_corrupt_t_0p0_0p2=0.8603 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.4382 out_g_norm=0.1898 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9985 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3392 init_gold_top10=0.7636 init_gold_top100=0.8143 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1437 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.8841 logit_acc_rollout_kept=0.8800 +step=5500 epoch=5500/6000 epoch_step=1/1 micro_steps=5500 elapsed=29.5s lr=2.000000e-03 loss=0.2449 loss_recon=0.2449 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9228 corrupt_frac=1.0000 acc_corrupt=0.9228 loss_corrupt=0.2449 wrong_frac=0.7930 init_acc_corrupt=0.1627 acc_corrupt_t_0p0_0p2=0.8630 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.4511 out_g_norm=0.1749 loss_all=0.2526 init_gold_top10=0.7666 init_gold_top100=0.8219 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1890 init_acc_rollout_kept=0.1196 logit_acc_rollout_applied=0.9171 logit_acc_rollout_kept=0.9350 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=29.5s lr=2.000000e-03 loss=0.2280 loss_recon=0.2280 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9274 corrupt_frac=1.0000 acc_corrupt=0.9274 loss_corrupt=0.2280 wrong_frac=0.7925 init_acc_corrupt=0.1629 acc_corrupt_t_0p0_0p2=0.8707 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.4633 out_g_norm=0.1667 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2745 init_gold_top10=0.7764 init_gold_top100=0.8351 rollout_applied_pos_frac=0.7578 init_acc_rollout_applied=0.1588 init_acc_rollout_kept=0.1223 logit_acc_rollout_applied=0.8995 logit_acc_rollout_kept=0.9500 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=29.4s lr=2.000000e-03 loss=0.2275 loss_recon=0.2275 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7473 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9278 corrupt_frac=1.0000 acc_corrupt=0.9278 loss_corrupt=0.2275 wrong_frac=0.7907 init_acc_corrupt=0.1656 acc_corrupt_t_0p0_0p2=0.8699 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.4753 out_g_norm=0.1628 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2194 init_gold_top10=0.8025 init_gold_top100=0.8433 rollout_applied_pos_frac=0.7656 init_acc_rollout_applied=0.1860 init_acc_rollout_kept=0.1329 logit_acc_rollout_applied=0.9352 logit_acc_rollout_kept=0.9421 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=29.5s lr=2.000000e-03 loss=0.2320 loss_recon=0.2320 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9280 corrupt_frac=1.0000 acc_corrupt=0.9280 loss_corrupt=0.2320 wrong_frac=0.7901 init_acc_corrupt=0.1663 acc_corrupt_t_0p0_0p2=0.8703 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.4854 out_g_norm=0.1675 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2149 init_gold_top10=0.8401 init_gold_top100=0.8787 rollout_applied_pos_frac=0.8125 init_acc_rollout_applied=0.2049 init_acc_rollout_kept=0.1695 logit_acc_rollout_applied=0.9277 logit_acc_rollout_kept=0.9351 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=29.3s lr=2.000000e-03 loss=0.2372 loss_recon=0.2372 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7442 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9240 corrupt_frac=1.0000 acc_corrupt=0.9240 loss_corrupt=0.2372 wrong_frac=0.7895 init_acc_corrupt=0.1668 acc_corrupt_t_0p0_0p2=0.8624 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.4874 out_g_norm=0.1675 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3191 init_gold_top10=0.7399 init_gold_top100=0.8028 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1811 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8865 logit_acc_rollout_kept=0.8959 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=29.4s lr=2.000000e-03 loss=0.2376 loss_recon=0.2376 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9264 corrupt_frac=1.0000 acc_corrupt=0.9264 loss_corrupt=0.2376 wrong_frac=0.7888 init_acc_corrupt=0.1680 acc_corrupt_t_0p0_0p2=0.8667 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.5026 out_g_norm=0.1718 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9992 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2906 init_gold_top10=0.7994 init_gold_top100=0.8592 rollout_applied_pos_frac=0.7891 init_acc_rollout_applied=0.1825 init_acc_rollout_kept=0.1499 logit_acc_rollout_applied=0.8927 logit_acc_rollout_kept=0.9896 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.75, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_time_mode": "fixed_steps", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_p75_s4_i32_outwdm1_ctx1024_sweep_selected_20260517_210705/latest.pt" +} +step=6100 epoch=6100/7000 epoch_step=1/1 micro_steps=6100 elapsed=30.3s lr=2.000000e-03 loss=0.2427 loss_recon=0.2427 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9229 corrupt_frac=1.0000 acc_corrupt=0.9229 loss_corrupt=0.2427 wrong_frac=0.7918 init_acc_corrupt=0.1642 acc_corrupt_t_0p0_0p2=0.8614 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5063 out_g_norm=0.1660 loss_all=0.1492 init_gold_top10=0.7566 init_gold_top100=0.8007 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.2014 init_acc_rollout_kept=0.0965 logit_acc_rollout_applied=0.9555 logit_acc_rollout_kept=0.9409 +step=6200 epoch=6200/7000 epoch_step=1/1 micro_steps=6200 elapsed=29.3s lr=2.000000e-03 loss=0.2184 loss_recon=0.2184 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9320 corrupt_frac=1.0000 acc_corrupt=0.9320 loss_corrupt=0.2184 wrong_frac=0.7893 init_acc_corrupt=0.1667 acc_corrupt_t_0p0_0p2=0.8767 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.5095 out_g_norm=0.1593 acc_corrupt_t_0p8_1p0=0.9984 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2125 init_gold_top10=0.7815 init_gold_top100=0.8294 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.1582 init_acc_rollout_kept=0.1576 logit_acc_rollout_applied=0.9131 logit_acc_rollout_kept=0.9865 +step=6300 epoch=6300/7000 epoch_step=1/1 micro_steps=6300 elapsed=29.4s lr=2.000000e-03 loss=0.2328 loss_recon=0.2328 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9269 corrupt_frac=1.0000 acc_corrupt=0.9269 loss_corrupt=0.2328 wrong_frac=0.7934 init_acc_corrupt=0.1622 acc_corrupt_t_0p0_0p2=0.8705 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5216 out_g_norm=0.1644 acc_corrupt_t_0p8_1p0=0.9984 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2667 init_gold_top10=0.7519 init_gold_top100=0.8177 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1684 init_acc_rollout_kept=0.1376 logit_acc_rollout_applied=0.9031 logit_acc_rollout_kept=0.9703 +step=6400 epoch=6400/7000 epoch_step=1/1 micro_steps=6400 elapsed=29.5s lr=2.000000e-03 loss=0.2344 loss_recon=0.2344 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9265 corrupt_frac=1.0000 acc_corrupt=0.9265 loss_corrupt=0.2344 wrong_frac=0.7914 init_acc_corrupt=0.1646 acc_corrupt_t_0p0_0p2=0.8679 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.5409 out_g_norm=0.1708 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3624 init_gold_top10=0.7512 init_gold_top100=0.8143 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.1437 init_acc_rollout_kept=0.1469 logit_acc_rollout_applied=0.8797 logit_acc_rollout_kept=0.9120 +W0517 21:40:17.104000 355614 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0517 21:40:17.106000 355614 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 355618 closing signal SIGTERM +W0517 21:40:17.106000 355614 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 355619 closing signal SIGTERM +W0517 21:40:17.107000 355614 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 355620 closing signal SIGTERM +W0517 21:40:17.107000 355614 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 355621 closing signal SIGTERM +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 355614 got signal: 15 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728.log new file mode 100644 index 0000000000000000000000000000000000000000..fd5ebc4b19d27873c001d4c11c67dc1625da786e --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728.log @@ -0,0 +1,1034 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 3, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=19.8s lr=2.000000e-03 loss=7.3417 loss_recon=7.3417 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3513 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3284 corrupt_frac=1.0000 acc_corrupt=0.3284 loss_corrupt=7.3417 wrong_frac=0.5028 init_acc_corrupt=0.4627 acc_corrupt_t_0p0_0p2=0.0466 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.1614 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.3279 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=0.4813 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=0.6352 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=1.0922 out_g_norm=1.0102 loss_all=6.7794 init_gold_top10=0.4656 init_gold_top100=0.6040 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.3928 init_acc_rollout_kept=0.4393 logit_acc_rollout_applied=0.2685 logit_acc_rollout_kept=0.2938 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=19.0s lr=2.000000e-03 loss=5.8034 loss_recon=5.8034 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3299 corrupt_frac=1.0000 acc_corrupt=0.3299 loss_corrupt=5.8034 wrong_frac=0.4984 init_acc_corrupt=0.4676 acc_corrupt_t_0p0_0p2=0.0524 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.1630 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.3295 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.4754 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=0.6253 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=3.4952 out_g_norm=1.3311 loss_all=5.0183 init_gold_top10=0.5079 init_gold_top100=0.6301 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.4755 init_acc_rollout_kept=0.4824 logit_acc_rollout_applied=0.3469 logit_acc_rollout_kept=0.3570 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=19.0s lr=2.000000e-03 loss=4.7182 loss_recon=4.7182 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3535 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3676 corrupt_frac=1.0000 acc_corrupt=0.3676 loss_corrupt=4.7182 wrong_frac=0.4985 init_acc_corrupt=0.4685 acc_corrupt_t_0p0_0p2=0.0557 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.1898 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.3639 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.5242 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=0.6974 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=5.5797 out_g_norm=0.5500 loss_all=4.3883 init_gold_top10=0.5102 init_gold_top100=0.6393 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.5089 init_acc_rollout_kept=0.4471 logit_acc_rollout_applied=0.4133 logit_acc_rollout_kept=0.3762 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=19.0s lr=2.000000e-03 loss=4.1225 loss_recon=4.1225 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3515 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4225 corrupt_frac=1.0000 acc_corrupt=0.4225 loss_corrupt=4.1225 wrong_frac=0.5016 init_acc_corrupt=0.4649 acc_corrupt_t_0p0_0p2=0.0583 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.2095 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.4200 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.6151 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=0.8157 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=7.1167 out_g_norm=0.2749 loss_all=3.9532 init_gold_top10=0.4850 init_gold_top100=0.6328 rollout_applied_pos_frac=0.3281 init_acc_rollout_applied=0.4657 init_acc_rollout_kept=0.4320 logit_acc_rollout_applied=0.4583 logit_acc_rollout_kept=0.4408 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=19.0s lr=2.000000e-03 loss=3.5466 loss_recon=3.5466 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4823 corrupt_frac=1.0000 acc_corrupt=0.4823 loss_corrupt=3.5466 wrong_frac=0.5023 init_acc_corrupt=0.4642 acc_corrupt_t_0p0_0p2=0.0594 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.2412 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=0.5048 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=0.7094 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=0.9033 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=8.4596 out_g_norm=0.2351 loss_all=3.3523 init_gold_top10=0.4858 init_gold_top100=0.6220 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.4617 init_acc_rollout_kept=0.4433 logit_acc_rollout_applied=0.4844 logit_acc_rollout_kept=0.4697 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=19.1s lr=2.000000e-03 loss=3.0821 loss_recon=3.0821 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3562 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4963 corrupt_frac=1.0000 acc_corrupt=0.4963 loss_corrupt=3.0821 wrong_frac=0.4987 init_acc_corrupt=0.4682 acc_corrupt_t_0p0_0p2=0.0622 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.2700 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=0.5248 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=0.7152 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=0.9049 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=9.7303 out_g_norm=0.2534 loss_all=2.4394 init_gold_top10=0.5858 init_gold_top100=0.7010 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5050 init_acc_rollout_kept=0.5727 logit_acc_rollout_applied=0.5326 logit_acc_rollout_kept=0.6010 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=18.9s lr=2.000000e-03 loss=2.7681 loss_recon=2.7681 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3412 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5034 corrupt_frac=1.0000 acc_corrupt=0.5034 loss_corrupt=2.7681 wrong_frac=0.4994 init_acc_corrupt=0.4679 acc_corrupt_t_0p0_0p2=0.0635 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.2807 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.5331 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.7234 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=0.9066 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=10.7428 out_g_norm=0.2922 loss_all=2.5966 init_gold_top10=0.5189 init_gold_top100=0.6915 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.4203 init_acc_rollout_kept=0.5121 logit_acc_rollout_applied=0.4614 logit_acc_rollout_kept=0.5496 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=19.0s lr=2.000000e-03 loss=2.3453 loss_recon=2.3453 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5308 corrupt_frac=1.0000 acc_corrupt=0.5308 loss_corrupt=2.3453 wrong_frac=0.5037 init_acc_corrupt=0.4633 acc_corrupt_t_0p0_0p2=0.0631 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.3065 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.5891 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.7800 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=0.9311 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=11.2908 out_g_norm=0.3703 loss_all=2.2007 init_gold_top10=0.5154 init_gold_top100=0.6697 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.4141 init_acc_rollout_kept=0.4622 logit_acc_rollout_applied=0.5173 logit_acc_rollout_kept=0.5575 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=19.1s lr=2.000000e-03 loss=1.8132 loss_recon=1.8132 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3546 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6146 corrupt_frac=1.0000 acc_corrupt=0.6146 loss_corrupt=1.8132 wrong_frac=0.4960 init_acc_corrupt=0.4740 acc_corrupt_t_0p0_0p2=0.0648 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.3780 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.7364 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.8940 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=0.9738 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=11.7769 out_g_norm=0.4581 loss_all=1.5519 init_gold_top10=0.5837 init_gold_top100=0.7187 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.4619 init_acc_rollout_kept=0.5155 logit_acc_rollout_applied=0.6272 logit_acc_rollout_kept=0.6871 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=19.0s lr=2.000000e-03 loss=1.5254 loss_recon=1.5254 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3534 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6756 corrupt_frac=1.0000 acc_corrupt=0.6756 loss_corrupt=1.5254 wrong_frac=0.4993 init_acc_corrupt=0.4741 acc_corrupt_t_0p0_0p2=0.0694 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.4889 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.8656 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=0.9668 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=0.9947 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=12.1609 out_g_norm=0.5459 loss_all=1.5778 init_gold_top10=0.5685 init_gold_top100=0.7045 rollout_applied_pos_frac=0.4141 init_acc_rollout_applied=0.5418 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.7332 logit_acc_rollout_kept=0.6367 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 3, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=20.2s lr=2.000000e-03 loss=1.3296 loss_recon=1.3296 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3513 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7155 corrupt_frac=1.0000 acc_corrupt=0.7155 loss_corrupt=1.3296 wrong_frac=0.5028 init_acc_corrupt=0.4747 acc_corrupt_t_0p0_0p2=0.0794 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.6046 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.9346 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=0.9896 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.4307 out_g_norm=0.5976 loss_all=1.1705 init_gold_top10=0.5527 init_gold_top100=0.6892 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.4722 init_acc_rollout_kept=0.4393 logit_acc_rollout_applied=0.7318 logit_acc_rollout_kept=0.7314 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=19.0s lr=2.000000e-03 loss=1.1354 loss_recon=1.1354 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7552 corrupt_frac=1.0000 acc_corrupt=0.7552 loss_corrupt=1.1354 wrong_frac=0.4984 init_acc_corrupt=0.4838 acc_corrupt_t_0p0_0p2=0.0990 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.7128 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.9679 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.9960 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.6239 out_g_norm=0.6265 loss_all=1.0650 init_gold_top10=0.5828 init_gold_top100=0.7012 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.5340 init_acc_rollout_kept=0.4824 logit_acc_rollout_applied=0.7880 logit_acc_rollout_kept=0.7683 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=19.0s lr=2.000000e-03 loss=0.9741 loss_recon=0.9741 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3535 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7882 corrupt_frac=1.0000 acc_corrupt=0.7882 loss_corrupt=0.9741 wrong_frac=0.4985 init_acc_corrupt=0.4873 acc_corrupt_t_0p0_0p2=0.1305 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.8078 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9844 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9982 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=12.7840 out_g_norm=0.6228 loss_all=0.8968 init_gold_top10=0.5980 init_gold_top100=0.7052 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.5557 init_acc_rollout_kept=0.4471 logit_acc_rollout_applied=0.8387 logit_acc_rollout_kept=0.7854 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=19.0s lr=2.000000e-03 loss=0.8769 loss_recon=0.8769 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3515 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8068 corrupt_frac=1.0000 acc_corrupt=0.8068 loss_corrupt=0.8769 wrong_frac=0.5016 init_acc_corrupt=0.4879 acc_corrupt_t_0p0_0p2=0.1718 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.8725 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.9927 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=12.9245 out_g_norm=0.6431 loss_all=0.7940 init_gold_top10=0.5796 init_gold_top100=0.7022 rollout_applied_pos_frac=0.3281 init_acc_rollout_applied=0.5728 init_acc_rollout_kept=0.4320 logit_acc_rollout_applied=0.8450 logit_acc_rollout_kept=0.8035 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=19.0s lr=2.000000e-03 loss=0.8124 loss_recon=0.8124 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8199 corrupt_frac=1.0000 acc_corrupt=0.8199 loss_corrupt=0.8124 wrong_frac=0.5023 init_acc_corrupt=0.4897 acc_corrupt_t_0p0_0p2=0.2024 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.9242 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=0.9964 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=13.0274 out_g_norm=0.6095 loss_all=0.7331 init_gold_top10=0.5547 init_gold_top100=0.6604 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5269 init_acc_rollout_kept=0.4433 logit_acc_rollout_applied=0.8746 logit_acc_rollout_kept=0.8114 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=19.1s lr=2.000000e-03 loss=0.6964 loss_recon=0.6964 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3562 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8418 corrupt_frac=1.0000 acc_corrupt=0.8418 loss_corrupt=0.6964 wrong_frac=0.4987 init_acc_corrupt=0.4972 acc_corrupt_t_0p0_0p2=0.2464 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.9545 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=0.9982 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=13.1072 out_g_norm=0.5888 loss_all=0.5103 init_gold_top10=0.6593 init_gold_top100=0.7448 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5605 init_acc_rollout_kept=0.5727 logit_acc_rollout_applied=0.8679 logit_acc_rollout_kept=0.8962 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=18.9s lr=2.000000e-03 loss=0.6638 loss_recon=0.6638 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3412 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8463 corrupt_frac=1.0000 acc_corrupt=0.8463 loss_corrupt=0.6638 wrong_frac=0.4994 init_acc_corrupt=0.4951 acc_corrupt_t_0p0_0p2=0.2739 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.9684 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.9989 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=13.1660 out_g_norm=0.6050 loss_all=0.8292 init_gold_top10=0.5939 init_gold_top100=0.7235 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.4727 init_acc_rollout_kept=0.5121 logit_acc_rollout_applied=0.7681 logit_acc_rollout_kept=0.8222 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=19.0s lr=2.000000e-03 loss=0.6150 loss_recon=0.6150 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8536 corrupt_frac=1.0000 acc_corrupt=0.8536 loss_corrupt=0.6150 wrong_frac=0.5037 init_acc_corrupt=0.4900 acc_corrupt_t_0p0_0p2=0.2985 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.9798 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=13.2069 out_g_norm=0.5504 loss_all=0.6526 init_gold_top10=0.5530 init_gold_top100=0.6933 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.4390 init_acc_rollout_kept=0.4622 logit_acc_rollout_applied=0.8218 logit_acc_rollout_kept=0.8440 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=19.1s lr=2.000000e-03 loss=0.5333 loss_recon=0.5333 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3546 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8671 corrupt_frac=1.0000 acc_corrupt=0.8671 loss_corrupt=0.5333 wrong_frac=0.4960 init_acc_corrupt=0.5035 acc_corrupt_t_0p0_0p2=0.3275 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.9857 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=13.2327 out_g_norm=0.5815 loss_all=0.3862 init_gold_top10=0.6463 init_gold_top100=0.7374 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5274 init_acc_rollout_kept=0.5155 logit_acc_rollout_applied=0.8912 logit_acc_rollout_kept=0.8996 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=19.0s lr=2.000000e-03 loss=0.4955 loss_recon=0.4955 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3534 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8700 corrupt_frac=1.0000 acc_corrupt=0.8700 loss_corrupt=0.4955 wrong_frac=0.4993 init_acc_corrupt=0.4977 acc_corrupt_t_0p0_0p2=0.3724 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.9888 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=13.2466 out_g_norm=0.5578 loss_all=0.6899 init_gold_top10=0.6007 init_gold_top100=0.7234 rollout_applied_pos_frac=0.4141 init_acc_rollout_applied=0.6060 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.8636 logit_acc_rollout_kept=0.7849 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 3, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=19.8s lr=2.000000e-03 loss=0.4583 loss_recon=0.4583 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3513 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8735 corrupt_frac=1.0000 acc_corrupt=0.8735 loss_corrupt=0.4583 wrong_frac=0.5028 init_acc_corrupt=0.4935 acc_corrupt_t_0p0_0p2=0.3975 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.9918 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=13.2591 out_g_norm=0.5308 loss_all=0.2927 init_gold_top10=0.5962 init_gold_top100=0.6969 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.5366 init_acc_rollout_kept=0.4393 logit_acc_rollout_applied=0.9168 logit_acc_rollout_kept=0.9161 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=19.0s lr=2.000000e-03 loss=0.4055 loss_recon=0.4055 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8848 corrupt_frac=1.0000 acc_corrupt=0.8848 loss_corrupt=0.4055 wrong_frac=0.4984 init_acc_corrupt=0.5000 acc_corrupt_t_0p0_0p2=0.4320 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9939 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=13.2548 out_g_norm=0.5596 loss_all=0.4308 init_gold_top10=0.6072 init_gold_top100=0.7113 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.5487 init_acc_rollout_kept=0.4824 logit_acc_rollout_applied=0.8832 logit_acc_rollout_kept=0.8671 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=19.1s lr=2.000000e-03 loss=0.3642 loss_recon=0.3642 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3535 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8920 corrupt_frac=1.0000 acc_corrupt=0.8920 loss_corrupt=0.3642 wrong_frac=0.4985 init_acc_corrupt=0.4995 acc_corrupt_t_0p0_0p2=0.4506 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.9954 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=13.2547 out_g_norm=0.5233 loss_all=0.3508 init_gold_top10=0.6184 init_gold_top100=0.7122 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.5912 init_acc_rollout_kept=0.4471 logit_acc_rollout_applied=0.9022 logit_acc_rollout_kept=0.8909 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=19.0s lr=2.000000e-03 loss=0.3392 loss_recon=0.3392 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3515 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8964 corrupt_frac=1.0000 acc_corrupt=0.8964 loss_corrupt=0.3392 wrong_frac=0.5016 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.4873 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9962 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=13.2416 out_g_norm=0.5315 loss_all=0.3230 init_gold_top10=0.6008 init_gold_top100=0.7086 rollout_applied_pos_frac=0.3281 init_acc_rollout_applied=0.6073 init_acc_rollout_kept=0.4320 logit_acc_rollout_applied=0.8929 logit_acc_rollout_kept=0.9091 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=19.0s lr=2.000000e-03 loss=0.3342 loss_recon=0.3342 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8955 corrupt_frac=1.0000 acc_corrupt=0.8955 loss_corrupt=0.3342 wrong_frac=0.5023 init_acc_corrupt=0.4972 acc_corrupt_t_0p0_0p2=0.4951 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.9976 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=13.2247 out_g_norm=0.4966 loss_all=0.2946 init_gold_top10=0.5647 init_gold_top100=0.6631 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5336 init_acc_rollout_kept=0.4433 logit_acc_rollout_applied=0.9112 logit_acc_rollout_kept=0.9019 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=19.1s lr=2.000000e-03 loss=0.2927 loss_recon=0.2927 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3562 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9071 corrupt_frac=1.0000 acc_corrupt=0.9071 loss_corrupt=0.2927 wrong_frac=0.4987 init_acc_corrupt=0.5030 acc_corrupt_t_0p0_0p2=0.5311 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.9981 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=13.1977 out_g_norm=0.4920 loss_all=0.2278 init_gold_top10=0.6797 init_gold_top100=0.7468 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5799 init_acc_rollout_kept=0.5727 logit_acc_rollout_applied=0.8991 logit_acc_rollout_kept=0.9380 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=18.9s lr=2.000000e-03 loss=0.2925 loss_recon=0.2925 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3412 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9053 corrupt_frac=1.0000 acc_corrupt=0.9053 loss_corrupt=0.2925 wrong_frac=0.4994 init_acc_corrupt=0.5003 acc_corrupt_t_0p0_0p2=0.5350 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.9984 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=13.1540 out_g_norm=0.4638 loss_all=0.4191 init_gold_top10=0.6214 init_gold_top100=0.7245 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.5020 init_acc_rollout_kept=0.5121 logit_acc_rollout_applied=0.8600 logit_acc_rollout_kept=0.8647 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=19.0s lr=2.000000e-03 loss=0.2855 loss_recon=0.2855 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9054 corrupt_frac=1.0000 acc_corrupt=0.9054 loss_corrupt=0.2855 wrong_frac=0.5037 init_acc_corrupt=0.4940 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.9986 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=13.1064 out_g_norm=0.4428 loss_all=0.3016 init_gold_top10=0.5783 init_gold_top100=0.6934 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.4441 init_acc_rollout_kept=0.4622 logit_acc_rollout_applied=0.8869 logit_acc_rollout_kept=0.9013 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=19.1s lr=2.000000e-03 loss=0.2654 loss_recon=0.2654 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3546 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9131 corrupt_frac=1.0000 acc_corrupt=0.9131 loss_corrupt=0.2654 wrong_frac=0.4960 init_acc_corrupt=0.5068 acc_corrupt_t_0p0_0p2=0.5513 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=13.0543 out_g_norm=0.4271 loss_all=0.1481 init_gold_top10=0.6606 init_gold_top100=0.7375 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5364 init_acc_rollout_kept=0.5155 logit_acc_rollout_applied=0.9541 logit_acc_rollout_kept=0.9501 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=19.0s lr=2.000000e-03 loss=0.2693 loss_recon=0.2693 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3534 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9110 corrupt_frac=1.0000 acc_corrupt=0.9110 loss_corrupt=0.2693 wrong_frac=0.4993 init_acc_corrupt=0.5007 acc_corrupt_t_0p0_0p2=0.5637 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=12.9986 out_g_norm=0.4138 loss_all=0.4644 init_gold_top10=0.6166 init_gold_top100=0.7234 rollout_applied_pos_frac=0.4141 init_acc_rollout_applied=0.6088 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.8838 logit_acc_rollout_kept=0.8375 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 3, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=19.8s lr=2.000000e-03 loss=0.2771 loss_recon=0.2771 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3513 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9082 corrupt_frac=1.0000 acc_corrupt=0.9082 loss_corrupt=0.2771 wrong_frac=0.5028 init_acc_corrupt=0.4959 acc_corrupt_t_0p0_0p2=0.5578 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.9438 out_g_norm=0.4133 loss_all=0.1770 init_gold_top10=0.6054 init_gold_top100=0.6969 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.5426 init_acc_rollout_kept=0.4393 logit_acc_rollout_applied=0.9321 logit_acc_rollout_kept=0.9395 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=19.0s lr=2.000000e-03 loss=0.2579 loss_recon=0.2579 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9130 corrupt_frac=1.0000 acc_corrupt=0.9130 loss_corrupt=0.2579 wrong_frac=0.4984 init_acc_corrupt=0.5023 acc_corrupt_t_0p0_0p2=0.5668 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.8972 out_g_norm=0.4063 loss_all=0.3037 init_gold_top10=0.6150 init_gold_top100=0.7113 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.5494 init_acc_rollout_kept=0.4824 logit_acc_rollout_applied=0.8931 logit_acc_rollout_kept=0.8928 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=19.0s lr=2.000000e-03 loss=0.2556 loss_recon=0.2556 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3535 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9136 corrupt_frac=1.0000 acc_corrupt=0.9136 loss_corrupt=0.2556 wrong_frac=0.4985 init_acc_corrupt=0.5011 acc_corrupt_t_0p0_0p2=0.5569 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=12.8626 out_g_norm=0.3628 loss_all=0.2428 init_gold_top10=0.6262 init_gold_top100=0.7122 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.5953 init_acc_rollout_kept=0.4471 logit_acc_rollout_applied=0.8962 logit_acc_rollout_kept=0.9286 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=19.0s lr=2.000000e-03 loss=0.2476 loss_recon=0.2476 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3515 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9170 corrupt_frac=1.0000 acc_corrupt=0.9170 loss_corrupt=0.2476 wrong_frac=0.5016 init_acc_corrupt=0.4994 acc_corrupt_t_0p0_0p2=0.5868 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=12.8240 out_g_norm=0.3630 loss_all=0.2311 init_gold_top10=0.6073 init_gold_top100=0.7086 rollout_applied_pos_frac=0.3281 init_acc_rollout_applied=0.6076 init_acc_rollout_kept=0.4320 logit_acc_rollout_applied=0.8999 logit_acc_rollout_kept=0.9337 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=18.9s lr=2.000000e-03 loss=0.2571 loss_recon=0.2571 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9129 corrupt_frac=1.0000 acc_corrupt=0.9129 loss_corrupt=0.2571 wrong_frac=0.5023 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.5775 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.7945 out_g_norm=0.3526 loss_all=0.2127 init_gold_top10=0.5645 init_gold_top100=0.6631 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5340 init_acc_rollout_kept=0.4433 logit_acc_rollout_applied=0.9209 logit_acc_rollout_kept=0.9339 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=19.1s lr=2.000000e-03 loss=0.2311 loss_recon=0.2311 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3562 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9220 corrupt_frac=1.0000 acc_corrupt=0.9220 loss_corrupt=0.2311 wrong_frac=0.4987 init_acc_corrupt=0.5040 acc_corrupt_t_0p0_0p2=0.6049 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=12.7752 out_g_norm=0.3081 loss_all=0.1592 init_gold_top10=0.6825 init_gold_top100=0.7468 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5806 init_acc_rollout_kept=0.5727 logit_acc_rollout_applied=0.9003 logit_acc_rollout_kept=0.9709 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=18.9s lr=2.000000e-03 loss=0.2345 loss_recon=0.2345 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3412 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9207 corrupt_frac=1.0000 acc_corrupt=0.9207 loss_corrupt=0.2345 wrong_frac=0.4994 init_acc_corrupt=0.5010 acc_corrupt_t_0p0_0p2=0.6098 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=12.7645 out_g_norm=0.2989 loss_all=0.3539 init_gold_top10=0.6224 init_gold_top100=0.7245 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.5012 init_acc_rollout_kept=0.5121 logit_acc_rollout_applied=0.8579 logit_acc_rollout_kept=0.8977 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=19.0s lr=2.000000e-03 loss=0.2347 loss_recon=0.2347 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9189 corrupt_frac=1.0000 acc_corrupt=0.9189 loss_corrupt=0.2347 wrong_frac=0.5037 init_acc_corrupt=0.4948 acc_corrupt_t_0p0_0p2=0.6001 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=12.7526 out_g_norm=0.3031 loss_all=0.2269 init_gold_top10=0.5784 init_gold_top100=0.6934 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.4589 init_acc_rollout_kept=0.4622 logit_acc_rollout_applied=0.9234 logit_acc_rollout_kept=0.9226 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=19.1s lr=2.000000e-03 loss=0.2202 loss_recon=0.2202 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3546 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9242 corrupt_frac=1.0000 acc_corrupt=0.9242 loss_corrupt=0.2202 wrong_frac=0.4960 init_acc_corrupt=0.5074 acc_corrupt_t_0p0_0p2=0.6082 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=12.7499 out_g_norm=0.3059 loss_all=0.1200 init_gold_top10=0.6612 init_gold_top100=0.7375 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5376 init_acc_rollout_kept=0.5155 logit_acc_rollout_applied=0.9567 logit_acc_rollout_kept=0.9620 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=19.0s lr=2.000000e-03 loss=0.2281 loss_recon=0.2281 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3534 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9216 corrupt_frac=1.0000 acc_corrupt=0.9216 loss_corrupt=0.2281 wrong_frac=0.4993 init_acc_corrupt=0.5014 acc_corrupt_t_0p0_0p2=0.6154 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=12.7635 out_g_norm=0.2545 loss_all=0.3671 init_gold_top10=0.6215 init_gold_top100=0.7234 rollout_applied_pos_frac=0.4141 init_acc_rollout_applied=0.6094 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.8936 logit_acc_rollout_kept=0.8557 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 3, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=19.8s lr=2.000000e-03 loss=0.2388 loss_recon=0.2388 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3513 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9192 corrupt_frac=1.0000 acc_corrupt=0.9192 loss_corrupt=0.2388 wrong_frac=0.5028 init_acc_corrupt=0.4964 acc_corrupt_t_0p0_0p2=0.6102 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.7665 out_g_norm=0.2375 loss_all=0.1860 init_gold_top10=0.6049 init_gold_top100=0.6969 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.5464 init_acc_rollout_kept=0.4393 logit_acc_rollout_applied=0.9101 logit_acc_rollout_kept=0.9438 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=19.0s lr=2.000000e-03 loss=0.2291 loss_recon=0.2291 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9221 corrupt_frac=1.0000 acc_corrupt=0.9221 loss_corrupt=0.2291 wrong_frac=0.4984 init_acc_corrupt=0.5025 acc_corrupt_t_0p0_0p2=0.6120 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.7798 out_g_norm=0.2175 loss_all=0.2915 init_gold_top10=0.6130 init_gold_top100=0.7113 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.5497 init_acc_rollout_kept=0.4824 logit_acc_rollout_applied=0.8851 logit_acc_rollout_kept=0.8986 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=19.0s lr=2.000000e-03 loss=0.2208 loss_recon=0.2208 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3535 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9235 corrupt_frac=1.0000 acc_corrupt=0.9235 loss_corrupt=0.2208 wrong_frac=0.4985 init_acc_corrupt=0.5016 acc_corrupt_t_0p0_0p2=0.6076 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=12.7930 out_g_norm=0.2144 loss_all=0.2136 init_gold_top10=0.6264 init_gold_top100=0.7122 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.5958 init_acc_rollout_kept=0.4471 logit_acc_rollout_applied=0.9151 logit_acc_rollout_kept=0.9313 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=19.0s lr=2.000000e-03 loss=0.2217 loss_recon=0.2217 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3515 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9240 corrupt_frac=1.0000 acc_corrupt=0.9240 loss_corrupt=0.2217 wrong_frac=0.5016 init_acc_corrupt=0.4998 acc_corrupt_t_0p0_0p2=0.6214 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=12.8076 out_g_norm=0.2126 loss_all=0.1889 init_gold_top10=0.6095 init_gold_top100=0.7086 rollout_applied_pos_frac=0.3281 init_acc_rollout_applied=0.6095 init_acc_rollout_kept=0.4320 logit_acc_rollout_applied=0.9396 logit_acc_rollout_kept=0.9343 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=18.9s lr=2.000000e-03 loss=0.2272 loss_recon=0.2272 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9222 corrupt_frac=1.0000 acc_corrupt=0.9222 loss_corrupt=0.2272 wrong_frac=0.5023 init_acc_corrupt=0.4989 acc_corrupt_t_0p0_0p2=0.6227 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.8130 out_g_norm=0.1987 loss_all=0.1815 init_gold_top10=0.5693 init_gold_top100=0.6631 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5343 init_acc_rollout_kept=0.4433 logit_acc_rollout_applied=0.9338 logit_acc_rollout_kept=0.9350 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=19.1s lr=2.000000e-03 loss=0.2009 loss_recon=0.2009 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3562 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9311 corrupt_frac=1.0000 acc_corrupt=0.9311 loss_corrupt=0.2009 wrong_frac=0.4987 init_acc_corrupt=0.5041 acc_corrupt_t_0p0_0p2=0.6509 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=12.8104 out_g_norm=0.1711 loss_all=0.1198 init_gold_top10=0.6808 init_gold_top100=0.7468 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5819 init_acc_rollout_kept=0.5727 logit_acc_rollout_applied=0.9010 logit_acc_rollout_kept=0.9871 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=18.8s lr=2.000000e-03 loss=0.2080 loss_recon=0.2080 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3412 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9287 corrupt_frac=1.0000 acc_corrupt=0.9287 loss_corrupt=0.2080 wrong_frac=0.4994 init_acc_corrupt=0.5013 acc_corrupt_t_0p0_0p2=0.6493 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=12.8228 out_g_norm=0.1683 loss_all=0.2800 init_gold_top10=0.6261 init_gold_top100=0.7245 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.5064 init_acc_rollout_kept=0.5121 logit_acc_rollout_applied=0.8821 logit_acc_rollout_kept=0.9122 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=19.0s lr=2.000000e-03 loss=0.2172 loss_recon=0.2172 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9244 corrupt_frac=1.0000 acc_corrupt=0.9244 loss_corrupt=0.2172 wrong_frac=0.5037 init_acc_corrupt=0.4952 acc_corrupt_t_0p0_0p2=0.6273 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=12.8338 out_g_norm=0.1678 loss_all=0.2473 init_gold_top10=0.5767 init_gold_top100=0.6934 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.4607 init_acc_rollout_kept=0.4622 logit_acc_rollout_applied=0.9255 logit_acc_rollout_kept=0.9273 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=19.1s lr=2.000000e-03 loss=0.2005 loss_recon=0.2005 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3546 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9313 corrupt_frac=1.0000 acc_corrupt=0.9313 loss_corrupt=0.2005 wrong_frac=0.4960 init_acc_corrupt=0.5079 acc_corrupt_t_0p0_0p2=0.6448 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=12.8373 out_g_norm=0.1631 loss_all=0.0820 init_gold_top10=0.6659 init_gold_top100=0.7375 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5385 init_acc_rollout_kept=0.5155 logit_acc_rollout_applied=0.9767 logit_acc_rollout_kept=0.9712 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=19.0s lr=2.000000e-03 loss=0.2097 loss_recon=0.2097 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3534 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9280 corrupt_frac=1.0000 acc_corrupt=0.9280 loss_corrupt=0.2097 wrong_frac=0.4993 init_acc_corrupt=0.5014 acc_corrupt_t_0p0_0p2=0.6470 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=12.8513 out_g_norm=0.1607 loss_all=0.3370 init_gold_top10=0.6228 init_gold_top100=0.7234 rollout_applied_pos_frac=0.4141 init_acc_rollout_applied=0.6155 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.9016 logit_acc_rollout_kept=0.8619 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014620.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014620.log new file mode 100644 index 0000000000000000000000000000000000000000..aa02e367d87bf46be6fe3c783832a8896213d342 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014620.log @@ -0,0 +1,196 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014620", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705.log new file mode 100644 index 0000000000000000000000000000000000000000..9cc6fbb0aecfa2d4eb8a52c172e97d6090e512b0 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705.log @@ -0,0 +1,1024 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=14.5s lr=2.000000e-03 loss=7.7225 loss_recon=7.7225 loss_meanflow=0.0000 mean_model_t=0.2070 mean_corrupt_t=0.2070 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3566 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0944 corrupt_frac=1.0000 acc_corrupt=0.0944 loss_corrupt=7.7225 wrong_frac=0.7930 init_acc_corrupt=0.1137 acc_corrupt_t_0p0_0p2=0.0502 corrupt_frac_t_0p0_0p2=0.5646 acc_corrupt_t_0p2_0p4=0.1261 corrupt_frac_t_0p2_0p4=0.3537 acc_corrupt_t_0p4_0p6=0.2510 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.3593 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=1.0070 out_g_norm=1.0880 acc_corrupt_t_0p8_1p0=0.4619 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.4683 init_gold_top10=0.1940 init_gold_top100=0.3863 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.1046 init_acc_rollout_kept=0.1284 logit_acc_rollout_applied=0.0924 logit_acc_rollout_kept=0.1034 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=13.5s lr=2.000000e-03 loss=7.0875 loss_recon=7.0875 loss_meanflow=0.0000 mean_model_t=0.2093 mean_corrupt_t=0.2093 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3467 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1033 corrupt_frac=1.0000 acc_corrupt=0.1033 loss_corrupt=7.0875 wrong_frac=0.7905 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.0558 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.1391 corrupt_frac_t_0p2_0p4=0.3600 acc_corrupt_t_0p4_0p6=0.2535 corrupt_frac_t_0p4_0p6=0.0760 acc_corrupt_t_0p6_0p8=0.3377 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=2.8606 out_g_norm=1.7741 acc_corrupt_t_0p8_1p0=0.2988 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.6883 init_gold_top10=0.2127 init_gold_top100=0.4090 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.1394 init_acc_rollout_kept=0.1165 logit_acc_rollout_applied=0.1224 logit_acc_rollout_kept=0.1105 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=13.5s lr=2.000000e-03 loss=6.4572 loss_recon=6.4572 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3452 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1128 corrupt_frac=1.0000 acc_corrupt=0.1128 loss_corrupt=6.4572 wrong_frac=0.7923 init_acc_corrupt=0.1157 acc_corrupt_t_0p0_0p2=0.0592 corrupt_frac_t_0p0_0p2=0.5677 acc_corrupt_t_0p2_0p4=0.1556 corrupt_frac_t_0p2_0p4=0.3483 acc_corrupt_t_0p4_0p6=0.2841 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.3918 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=4.3503 out_g_norm=1.3203 acc_corrupt_t_0p8_1p0=0.5454 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.2518 init_gold_top10=0.1851 init_gold_top100=0.3919 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.0706 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.0955 logit_acc_rollout_kept=0.1151 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=13.6s lr=2.000000e-03 loss=5.9689 loss_recon=5.9689 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1251 corrupt_frac=1.0000 acc_corrupt=0.1251 loss_corrupt=5.9689 wrong_frac=0.7904 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.0635 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.1711 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.3157 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.4369 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=5.5056 out_g_norm=0.5242 acc_corrupt_t_0p8_1p0=0.5684 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.7489 init_gold_top10=0.1987 init_gold_top100=0.4110 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1260 init_acc_rollout_kept=0.1193 logit_acc_rollout_applied=0.1379 logit_acc_rollout_kept=0.1298 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=13.6s lr=2.000000e-03 loss=5.4717 loss_recon=5.4717 loss_meanflow=0.0000 mean_model_t=0.2075 mean_corrupt_t=0.2075 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3543 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1360 corrupt_frac=1.0000 acc_corrupt=0.1360 loss_corrupt=5.4717 wrong_frac=0.7924 init_acc_corrupt=0.1157 acc_corrupt_t_0p0_0p2=0.0675 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.1901 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.3503 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.4852 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=6.7726 out_g_norm=0.4126 acc_corrupt_t_0p8_1p0=0.6328 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.1699 init_gold_top10=0.2023 init_gold_top100=0.4501 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1088 init_acc_rollout_kept=0.1267 logit_acc_rollout_applied=0.1348 logit_acc_rollout_kept=0.1538 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=13.5s lr=2.000000e-03 loss=4.8275 loss_recon=4.8275 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1514 corrupt_frac=1.0000 acc_corrupt=0.1514 loss_corrupt=4.8275 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.0729 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.2141 corrupt_frac_t_0p2_0p4=0.3601 acc_corrupt_t_0p4_0p6=0.3970 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.5683 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=8.0690 out_g_norm=0.4267 acc_corrupt_t_0p8_1p0=0.6172 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.3866 init_gold_top10=0.2056 init_gold_top100=0.5031 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.1209 init_acc_rollout_kept=0.1135 logit_acc_rollout_applied=0.1647 logit_acc_rollout_kept=0.1675 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=13.5s lr=2.000000e-03 loss=4.2103 loss_recon=4.2103 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1766 corrupt_frac=1.0000 acc_corrupt=0.1766 loss_corrupt=4.2103 wrong_frac=0.7918 init_acc_corrupt=0.1171 acc_corrupt_t_0p0_0p2=0.0787 corrupt_frac_t_0p0_0p2=0.5605 acc_corrupt_t_0p2_0p4=0.2543 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=0.4862 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.6680 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=9.2577 out_g_norm=0.4974 acc_corrupt_t_0p8_1p0=0.8032 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.9837 init_gold_top10=0.2096 init_gold_top100=0.5203 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.1317 init_acc_rollout_kept=0.1155 logit_acc_rollout_applied=0.2035 logit_acc_rollout_kept=0.1879 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=13.5s lr=2.000000e-03 loss=3.7190 loss_recon=3.7190 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1987 corrupt_frac=1.0000 acc_corrupt=0.1987 loss_corrupt=3.7190 wrong_frac=0.7914 init_acc_corrupt=0.1172 acc_corrupt_t_0p0_0p2=0.0885 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.2933 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.5263 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.6865 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=10.1103 out_g_norm=0.6562 loss_all=3.4861 init_gold_top10=0.2151 init_gold_top100=0.5456 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1216 init_acc_rollout_kept=0.1331 logit_acc_rollout_applied=0.2087 logit_acc_rollout_kept=0.2242 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=13.6s lr=2.000000e-03 loss=3.3010 loss_recon=3.3010 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3524 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2325 corrupt_frac=1.0000 acc_corrupt=0.2325 loss_corrupt=3.3010 wrong_frac=0.7903 init_acc_corrupt=0.1189 acc_corrupt_t_0p0_0p2=0.1019 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.3438 corrupt_frac_t_0p2_0p4=0.3675 acc_corrupt_t_0p4_0p6=0.5826 corrupt_frac_t_0p4_0p6=0.0754 out_w_norm=10.6246 out_g_norm=0.8460 acc_corrupt_t_0p6_0p8=0.7352 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.8434 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.1073 init_gold_top10=0.2286 init_gold_top100=0.5535 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1193 init_acc_rollout_kept=0.1251 logit_acc_rollout_applied=0.2425 logit_acc_rollout_kept=0.2695 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=13.5s lr=2.000000e-03 loss=2.9096 loss_recon=2.9096 loss_meanflow=0.0000 mean_model_t=0.2090 mean_corrupt_t=0.2090 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2852 corrupt_frac=1.0000 acc_corrupt=0.2852 loss_corrupt=2.9096 wrong_frac=0.7909 init_acc_corrupt=0.1190 acc_corrupt_t_0p0_0p2=0.1272 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.4349 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.6706 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.8008 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=10.9832 out_g_norm=1.0880 acc_corrupt_t_0p8_1p0=0.8379 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.7501 init_gold_top10=0.2276 init_gold_top100=0.5129 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.0947 init_acc_rollout_kept=0.1007 logit_acc_rollout_applied=0.2757 logit_acc_rollout_kept=0.3128 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=14.5s lr=2.000000e-03 loss=2.5661 loss_recon=2.5661 loss_meanflow=0.0000 mean_model_t=0.2070 mean_corrupt_t=0.2070 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3566 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3462 corrupt_frac=1.0000 acc_corrupt=0.3462 loss_corrupt=2.5661 wrong_frac=0.7930 init_acc_corrupt=0.1172 acc_corrupt_t_0p0_0p2=0.1600 corrupt_frac_t_0p0_0p2=0.5646 acc_corrupt_t_0p2_0p4=0.5433 corrupt_frac_t_0p2_0p4=0.3537 acc_corrupt_t_0p4_0p6=0.7710 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8578 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=11.2564 out_g_norm=1.2812 acc_corrupt_t_0p8_1p0=0.9385 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.4453 init_gold_top10=0.2908 init_gold_top100=0.5703 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.1161 init_acc_rollout_kept=0.1284 logit_acc_rollout_applied=0.3787 logit_acc_rollout_kept=0.3859 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=13.5s lr=2.000000e-03 loss=2.2222 loss_recon=2.2222 loss_meanflow=0.0000 mean_model_t=0.2093 mean_corrupt_t=0.2093 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3467 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4209 corrupt_frac=1.0000 acc_corrupt=0.4209 loss_corrupt=2.2222 wrong_frac=0.7905 init_acc_corrupt=0.1209 acc_corrupt_t_0p0_0p2=0.1983 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.6590 corrupt_frac_t_0p2_0p4=0.3600 acc_corrupt_t_0p4_0p6=0.8595 corrupt_frac_t_0p4_0p6=0.0760 acc_corrupt_t_0p6_0p8=0.9181 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=11.5179 out_g_norm=1.4931 acc_corrupt_t_0p8_1p0=0.9355 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.7875 init_gold_top10=0.3507 init_gold_top100=0.5781 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.1554 init_acc_rollout_kept=0.1165 logit_acc_rollout_applied=0.5391 logit_acc_rollout_kept=0.4889 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=13.5s lr=2.000000e-03 loss=1.9908 loss_recon=1.9908 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3452 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4756 corrupt_frac=1.0000 acc_corrupt=0.4756 loss_corrupt=1.9908 wrong_frac=0.7923 init_acc_corrupt=0.1205 acc_corrupt_t_0p0_0p2=0.2363 corrupt_frac_t_0p0_0p2=0.5677 acc_corrupt_t_0p2_0p4=0.7568 corrupt_frac_t_0p2_0p4=0.3483 acc_corrupt_t_0p4_0p6=0.9233 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.9486 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=11.7426 out_g_norm=1.6005 acc_corrupt_t_0p8_1p0=0.9565 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.8900 init_gold_top10=0.3031 init_gold_top100=0.5160 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.0895 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.5288 logit_acc_rollout_kept=0.4775 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=13.6s lr=2.000000e-03 loss=1.7732 loss_recon=1.7732 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5297 corrupt_frac=1.0000 acc_corrupt=0.5297 loss_corrupt=1.7732 wrong_frac=0.7904 init_acc_corrupt=0.1238 acc_corrupt_t_0p0_0p2=0.2730 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.8254 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9553 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.9705 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=11.8904 out_g_norm=1.7736 acc_corrupt_t_0p8_1p0=0.9485 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.6616 init_gold_top10=0.3314 init_gold_top100=0.5167 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1414 init_acc_rollout_kept=0.1193 logit_acc_rollout_applied=0.6786 logit_acc_rollout_kept=0.5277 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=13.6s lr=2.000000e-03 loss=1.6078 loss_recon=1.6078 loss_meanflow=0.0000 mean_model_t=0.2075 mean_corrupt_t=0.2075 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3543 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5728 corrupt_frac=1.0000 acc_corrupt=0.5728 loss_corrupt=1.6078 wrong_frac=0.7924 init_acc_corrupt=0.1231 acc_corrupt_t_0p0_0p2=0.3100 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.8863 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9791 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9870 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=12.0140 out_g_norm=1.7297 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.6049 init_gold_top10=0.3586 init_gold_top100=0.5673 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1354 init_acc_rollout_kept=0.1267 logit_acc_rollout_applied=0.5946 logit_acc_rollout_kept=0.5775 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=13.6s lr=2.000000e-03 loss=1.4398 loss_recon=1.4398 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6148 corrupt_frac=1.0000 acc_corrupt=0.6148 loss_corrupt=1.4398 wrong_frac=0.7919 init_acc_corrupt=0.1239 acc_corrupt_t_0p0_0p2=0.3616 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.9233 corrupt_frac_t_0p2_0p4=0.3601 acc_corrupt_t_0p4_0p6=0.9885 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9892 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.1155 out_g_norm=1.7083 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2419 init_gold_top10=0.3671 init_gold_top100=0.5536 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.1364 init_acc_rollout_kept=0.1135 logit_acc_rollout_applied=0.6601 logit_acc_rollout_kept=0.6458 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=13.5s lr=2.000000e-03 loss=1.3160 loss_recon=1.3160 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6473 corrupt_frac=1.0000 acc_corrupt=0.6473 loss_corrupt=1.3160 wrong_frac=0.7918 init_acc_corrupt=0.1264 acc_corrupt_t_0p0_0p2=0.4046 corrupt_frac_t_0p0_0p2=0.5605 acc_corrupt_t_0p2_0p4=0.9485 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=0.9930 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9935 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.1736 out_g_norm=1.7850 acc_corrupt_t_0p8_1p0=0.9946 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3934 init_gold_top10=0.3819 init_gold_top100=0.5598 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.1794 init_acc_rollout_kept=0.1155 logit_acc_rollout_applied=0.7141 logit_acc_rollout_kept=0.6128 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=13.5s lr=2.000000e-03 loss=1.1661 loss_recon=1.1661 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6826 corrupt_frac=1.0000 acc_corrupt=0.6826 loss_corrupt=1.1661 wrong_frac=0.7914 init_acc_corrupt=0.1274 acc_corrupt_t_0p0_0p2=0.4589 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.9632 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9957 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.9944 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2279 out_g_norm=1.7180 loss_all=1.1713 init_gold_top10=0.3758 init_gold_top100=0.5692 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1465 init_acc_rollout_kept=0.1331 logit_acc_rollout_applied=0.7110 logit_acc_rollout_kept=0.6753 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=13.6s lr=2.000000e-03 loss=1.0032 loss_recon=1.0032 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3524 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7219 corrupt_frac=1.0000 acc_corrupt=0.7219 loss_corrupt=1.0032 wrong_frac=0.7903 init_acc_corrupt=0.1292 acc_corrupt_t_0p0_0p2=0.5132 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9708 corrupt_frac_t_0p2_0p4=0.3675 acc_corrupt_t_0p4_0p6=0.9962 corrupt_frac_t_0p4_0p6=0.0754 out_w_norm=12.2700 out_g_norm=1.8058 acc_corrupt_t_0p6_0p8=0.9950 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8831 init_gold_top10=0.3894 init_gold_top100=0.5688 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1433 init_acc_rollout_kept=0.1251 logit_acc_rollout_applied=0.7317 logit_acc_rollout_kept=0.7577 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=13.6s lr=2.000000e-03 loss=0.7741 loss_recon=0.7741 loss_meanflow=0.0000 mean_model_t=0.2090 mean_corrupt_t=0.2090 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.7741 wrong_frac=0.7909 init_acc_corrupt=0.1300 acc_corrupt_t_0p0_0p2=0.6133 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.9805 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9966 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9938 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.3161 out_g_norm=1.6323 acc_corrupt_t_0p8_1p0=0.9678 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6070 init_gold_top10=0.3857 init_gold_top100=0.5204 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.1180 init_acc_rollout_kept=0.1007 logit_acc_rollout_applied=0.8249 logit_acc_rollout_kept=0.8256 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=14.7s lr=2.000000e-03 loss=0.6071 loss_recon=0.6071 loss_meanflow=0.0000 mean_model_t=0.2070 mean_corrupt_t=0.2070 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3566 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8207 corrupt_frac=1.0000 acc_corrupt=0.8207 loss_corrupt=0.6071 wrong_frac=0.7930 init_acc_corrupt=0.1295 acc_corrupt_t_0p0_0p2=0.6896 corrupt_frac_t_0p0_0p2=0.5646 acc_corrupt_t_0p2_0p4=0.9893 corrupt_frac_t_0p2_0p4=0.3537 acc_corrupt_t_0p4_0p6=0.9978 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9961 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.3759 out_g_norm=1.3563 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5944 init_gold_top10=0.4541 init_gold_top100=0.5788 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.1540 init_acc_rollout_kept=0.1284 logit_acc_rollout_applied=0.8264 logit_acc_rollout_kept=0.8117 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=13.8s lr=2.000000e-03 loss=0.5238 loss_recon=0.5238 loss_meanflow=0.0000 mean_model_t=0.2093 mean_corrupt_t=0.2093 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3467 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8450 corrupt_frac=1.0000 acc_corrupt=0.8450 loss_corrupt=0.5238 wrong_frac=0.7905 init_acc_corrupt=0.1334 acc_corrupt_t_0p0_0p2=0.7252 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9936 corrupt_frac_t_0p2_0p4=0.3600 acc_corrupt_t_0p4_0p6=0.9980 corrupt_frac_t_0p4_0p6=0.0760 acc_corrupt_t_0p6_0p8=0.9972 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.3868 out_g_norm=1.3063 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3427 init_gold_top10=0.4658 init_gold_top100=0.5809 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.2087 init_acc_rollout_kept=0.1165 logit_acc_rollout_applied=0.8794 logit_acc_rollout_kept=0.9044 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=13.8s lr=2.000000e-03 loss=0.4988 loss_recon=0.4988 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3452 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8515 corrupt_frac=1.0000 acc_corrupt=0.8515 loss_corrupt=0.4988 wrong_frac=0.7923 init_acc_corrupt=0.1331 acc_corrupt_t_0p0_0p2=0.7407 corrupt_frac_t_0p0_0p2=0.5677 acc_corrupt_t_0p2_0p4=0.9965 corrupt_frac_t_0p2_0p4=0.3483 acc_corrupt_t_0p4_0p6=0.9987 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.3557 out_g_norm=1.2158 acc_corrupt_t_0p8_1p0=0.9985 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4199 init_gold_top10=0.3863 init_gold_top100=0.5180 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1357 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.8559 logit_acc_rollout_kept=0.8958 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=13.9s lr=2.000000e-03 loss=0.4452 loss_recon=0.4452 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8646 corrupt_frac=1.0000 acc_corrupt=0.8646 loss_corrupt=0.4452 wrong_frac=0.7904 init_acc_corrupt=0.1363 acc_corrupt_t_0p0_0p2=0.7581 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.9976 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9989 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.9987 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=12.3157 out_g_norm=1.0273 acc_corrupt_t_0p8_1p0=0.9968 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5021 init_gold_top10=0.3939 init_gold_top100=0.5171 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1624 init_acc_rollout_kept=0.1193 logit_acc_rollout_applied=0.9255 logit_acc_rollout_kept=0.8112 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=13.9s lr=2.000000e-03 loss=0.4050 loss_recon=0.4050 loss_meanflow=0.0000 mean_model_t=0.2075 mean_corrupt_t=0.2075 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3543 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8737 corrupt_frac=1.0000 acc_corrupt=0.8737 loss_corrupt=0.4050 wrong_frac=0.7924 init_acc_corrupt=0.1349 acc_corrupt_t_0p0_0p2=0.7741 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9992 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=12.2846 out_g_norm=0.8842 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5535 init_gold_top10=0.4440 init_gold_top100=0.5682 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1681 init_acc_rollout_kept=0.1267 logit_acc_rollout_applied=0.8005 logit_acc_rollout_kept=0.8542 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=13.8s lr=2.000000e-03 loss=0.3667 loss_recon=0.3667 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8866 corrupt_frac=1.0000 acc_corrupt=0.8866 loss_corrupt=0.3667 wrong_frac=0.7919 init_acc_corrupt=0.1346 acc_corrupt_t_0p0_0p2=0.7975 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.3601 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9992 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.2575 out_g_norm=0.8410 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2945 init_gold_top10=0.4462 init_gold_top100=0.5540 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.1552 init_acc_rollout_kept=0.1135 logit_acc_rollout_applied=0.8925 logit_acc_rollout_kept=0.9056 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=13.8s lr=2.000000e-03 loss=0.3555 loss_recon=0.3555 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8894 corrupt_frac=1.0000 acc_corrupt=0.8894 loss_corrupt=0.3555 wrong_frac=0.7918 init_acc_corrupt=0.1361 acc_corrupt_t_0p0_0p2=0.8030 corrupt_frac_t_0p0_0p2=0.5605 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2373 out_g_norm=0.7415 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4078 init_gold_top10=0.4547 init_gold_top100=0.5600 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.2303 init_acc_rollout_kept=0.1155 logit_acc_rollout_applied=0.8906 logit_acc_rollout_kept=0.8987 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=13.8s lr=2.000000e-03 loss=0.3279 loss_recon=0.3279 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8969 corrupt_frac=1.0000 acc_corrupt=0.8969 loss_corrupt=0.3279 wrong_frac=0.7914 init_acc_corrupt=0.1362 acc_corrupt_t_0p0_0p2=0.8168 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2266 out_g_norm=0.7464 loss_all=0.3466 init_gold_top10=0.4644 init_gold_top100=0.5694 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1603 init_acc_rollout_kept=0.1331 logit_acc_rollout_applied=0.8982 logit_acc_rollout_kept=0.8894 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=13.8s lr=2.000000e-03 loss=0.3187 loss_recon=0.3187 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3524 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9008 corrupt_frac=1.0000 acc_corrupt=0.9008 loss_corrupt=0.3187 wrong_frac=0.7903 init_acc_corrupt=0.1369 acc_corrupt_t_0p0_0p2=0.8194 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3675 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0754 out_w_norm=12.2105 out_g_norm=0.6139 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3607 init_gold_top10=0.4493 init_gold_top100=0.5688 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1612 init_acc_rollout_kept=0.1251 logit_acc_rollout_applied=0.8823 logit_acc_rollout_kept=0.8900 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=13.7s lr=2.000000e-03 loss=0.3103 loss_recon=0.3103 loss_meanflow=0.0000 mean_model_t=0.2090 mean_corrupt_t=0.2090 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9017 corrupt_frac=1.0000 acc_corrupt=0.9017 loss_corrupt=0.3103 wrong_frac=0.7909 init_acc_corrupt=0.1370 acc_corrupt_t_0p0_0p2=0.8231 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.2031 out_g_norm=0.6367 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2591 init_gold_top10=0.4073 init_gold_top100=0.5204 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.1303 init_acc_rollout_kept=0.1007 logit_acc_rollout_applied=0.8910 logit_acc_rollout_kept=0.9292 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=14.5s lr=2.000000e-03 loss=0.2988 loss_recon=0.2988 loss_meanflow=0.0000 mean_model_t=0.2070 mean_corrupt_t=0.2070 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3566 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9057 corrupt_frac=1.0000 acc_corrupt=0.9057 loss_corrupt=0.2988 wrong_frac=0.7930 init_acc_corrupt=0.1349 acc_corrupt_t_0p0_0p2=0.8331 corrupt_frac_t_0p0_0p2=0.5646 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3537 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.2055 out_g_norm=0.5154 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4181 init_gold_top10=0.4698 init_gold_top100=0.5788 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.1684 init_acc_rollout_kept=0.1284 logit_acc_rollout_applied=0.8597 logit_acc_rollout_kept=0.8469 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=13.5s lr=2.000000e-03 loss=0.2808 loss_recon=0.2808 loss_meanflow=0.0000 mean_model_t=0.2093 mean_corrupt_t=0.2093 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3467 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9119 corrupt_frac=1.0000 acc_corrupt=0.9119 loss_corrupt=0.2808 wrong_frac=0.7905 init_acc_corrupt=0.1374 acc_corrupt_t_0p0_0p2=0.8414 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3600 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0760 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.2055 out_g_norm=0.6080 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1874 init_gold_top10=0.4693 init_gold_top100=0.5809 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.2221 init_acc_rollout_kept=0.1165 logit_acc_rollout_applied=0.8865 logit_acc_rollout_kept=0.9701 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=13.5s lr=2.000000e-03 loss=0.2906 loss_recon=0.2906 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3452 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9074 corrupt_frac=1.0000 acc_corrupt=0.9074 loss_corrupt=0.2906 wrong_frac=0.7923 init_acc_corrupt=0.1364 acc_corrupt_t_0p0_0p2=0.8369 corrupt_frac_t_0p0_0p2=0.5677 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3483 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.2204 out_g_norm=0.4741 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3067 init_gold_top10=0.4022 init_gold_top100=0.5180 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1471 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.9373 logit_acc_rollout_kept=0.8873 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=13.6s lr=2.000000e-03 loss=0.2800 loss_recon=0.2800 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9107 corrupt_frac=1.0000 acc_corrupt=0.9107 loss_corrupt=0.2800 wrong_frac=0.7904 init_acc_corrupt=0.1393 acc_corrupt_t_0p0_0p2=0.8396 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=12.2315 out_g_norm=0.4718 acc_corrupt_t_0p8_1p0=0.9988 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4144 init_gold_top10=0.3943 init_gold_top100=0.5171 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1679 init_acc_rollout_kept=0.1193 logit_acc_rollout_applied=0.9457 logit_acc_rollout_kept=0.8529 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=13.6s lr=2.000000e-03 loss=0.2717 loss_recon=0.2717 loss_meanflow=0.0000 mean_model_t=0.2075 mean_corrupt_t=0.2075 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3543 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9142 corrupt_frac=1.0000 acc_corrupt=0.9142 loss_corrupt=0.2717 wrong_frac=0.7924 init_acc_corrupt=0.1371 acc_corrupt_t_0p0_0p2=0.8463 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=12.2636 out_g_norm=0.4406 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3554 init_gold_top10=0.4580 init_gold_top100=0.5682 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1740 init_acc_rollout_kept=0.1267 logit_acc_rollout_applied=0.8787 logit_acc_rollout_kept=0.8976 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=13.5s lr=2.000000e-03 loss=0.2541 loss_recon=0.2541 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3501 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9199 corrupt_frac=1.0000 acc_corrupt=0.9199 loss_corrupt=0.2541 wrong_frac=0.7919 init_acc_corrupt=0.1365 acc_corrupt_t_0p0_0p2=0.8567 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3601 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.2850 out_g_norm=0.3934 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2026 init_gold_top10=0.4512 init_gold_top100=0.5540 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.1588 init_acc_rollout_kept=0.1135 logit_acc_rollout_applied=0.9273 logit_acc_rollout_kept=0.9316 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=13.5s lr=2.000000e-03 loss=0.2640 loss_recon=0.2640 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3470 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9161 corrupt_frac=1.0000 acc_corrupt=0.9161 loss_corrupt=0.2640 wrong_frac=0.7918 init_acc_corrupt=0.1378 acc_corrupt_t_0p0_0p2=0.8504 corrupt_frac_t_0p0_0p2=0.5605 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.3074 out_g_norm=0.3603 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2453 init_gold_top10=0.4635 init_gold_top100=0.5600 rollout_applied_pos_frac=0.3516 init_acc_rollout_applied=0.2373 init_acc_rollout_kept=0.1155 logit_acc_rollout_applied=0.9115 logit_acc_rollout_kept=0.9218 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=13.5s lr=2.000000e-03 loss=0.2382 loss_recon=0.2382 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3430 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9229 corrupt_frac=1.0000 acc_corrupt=0.9229 loss_corrupt=0.2382 wrong_frac=0.7914 init_acc_corrupt=0.1377 acc_corrupt_t_0p0_0p2=0.8628 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.3346 out_g_norm=0.3186 loss_all=0.2397 init_gold_top10=0.4725 init_gold_top100=0.5694 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1680 init_acc_rollout_kept=0.1331 logit_acc_rollout_applied=0.9598 logit_acc_rollout_kept=0.9054 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=13.6s lr=2.000000e-03 loss=0.2536 loss_recon=0.2536 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3524 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9199 corrupt_frac=1.0000 acc_corrupt=0.9199 loss_corrupt=0.2536 wrong_frac=0.7903 init_acc_corrupt=0.1382 acc_corrupt_t_0p0_0p2=0.8541 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3675 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0754 out_w_norm=12.3579 out_g_norm=0.3368 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2855 init_gold_top10=0.4527 init_gold_top100=0.5688 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1643 init_acc_rollout_kept=0.1251 logit_acc_rollout_applied=0.8880 logit_acc_rollout_kept=0.9133 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=13.5s lr=2.000000e-03 loss=0.2502 loss_recon=0.2502 loss_meanflow=0.0000 mean_model_t=0.2090 mean_corrupt_t=0.2090 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9205 corrupt_frac=1.0000 acc_corrupt=0.9205 loss_corrupt=0.2502 wrong_frac=0.7909 init_acc_corrupt=0.1380 acc_corrupt_t_0p0_0p2=0.8569 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.3747 out_g_norm=0.3521 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2211 init_gold_top10=0.4130 init_gold_top100=0.5204 rollout_applied_pos_frac=0.3047 init_acc_rollout_applied=0.1318 init_acc_rollout_kept=0.1007 logit_acc_rollout_applied=0.9164 logit_acc_rollout_kept=0.9363 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_s", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_tradeoff_p35_unif0_0p25_outwdm1_ctx1024_tradeoff_dual_20260517_225705/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=14.7s lr=2.000000e-03 loss=0.2475 loss_recon=0.2475 loss_meanflow=0.0000 mean_model_t=0.2070 mean_corrupt_t=0.2070 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3566 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9226 corrupt_frac=1.0000 acc_corrupt=0.9226 loss_corrupt=0.2475 wrong_frac=0.7930 init_acc_corrupt=0.1356 acc_corrupt_t_0p0_0p2=0.8630 corrupt_frac_t_0p0_0p2=0.5646 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3537 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.3913 out_g_norm=0.2801 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3210 init_gold_top10=0.4741 init_gold_top100=0.5788 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.1707 init_acc_rollout_kept=0.1284 logit_acc_rollout_applied=0.8816 logit_acc_rollout_kept=0.8989 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=13.8s lr=2.000000e-03 loss=0.2207 loss_recon=0.2207 loss_meanflow=0.0000 mean_model_t=0.2093 mean_corrupt_t=0.2093 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3467 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9305 corrupt_frac=1.0000 acc_corrupt=0.9305 loss_corrupt=0.2207 wrong_frac=0.7905 init_acc_corrupt=0.1383 acc_corrupt_t_0p0_0p2=0.8749 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3600 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0760 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=12.4189 out_g_norm=0.2572 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1595 init_gold_top10=0.4790 init_gold_top100=0.5809 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.2249 init_acc_rollout_kept=0.1165 logit_acc_rollout_applied=0.9164 logit_acc_rollout_kept=0.9773 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=13.8s lr=2.000000e-03 loss=0.2405 loss_recon=0.2405 loss_meanflow=0.0000 mean_model_t=0.2077 mean_corrupt_t=0.2077 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3452 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9226 corrupt_frac=1.0000 acc_corrupt=0.9226 loss_corrupt=0.2405 wrong_frac=0.7923 init_acc_corrupt=0.1372 acc_corrupt_t_0p0_0p2=0.8638 corrupt_frac_t_0p0_0p2=0.5677 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3483 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0743 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.4505 out_g_norm=0.2480 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2279 init_gold_top10=0.3981 init_gold_top100=0.5180 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1497 init_acc_rollout_kept=0.1125 logit_acc_rollout_applied=0.9164 logit_acc_rollout_kept=0.9304 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=13.8s lr=2.000000e-03 loss=0.2299 loss_recon=0.2299 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3509 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9278 corrupt_frac=1.0000 acc_corrupt=0.9278 loss_corrupt=0.2299 wrong_frac=0.7904 init_acc_corrupt=0.1401 acc_corrupt_t_0p0_0p2=0.8702 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0138 out_w_norm=12.4714 out_g_norm=0.2465 acc_corrupt_t_0p8_1p0=0.9983 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5788 init_gold_top10=0.3953 init_gold_top100=0.5171 rollout_applied_pos_frac=0.2891 init_acc_rollout_applied=0.1687 init_acc_rollout_kept=0.1193 logit_acc_rollout_applied=0.9465 logit_acc_rollout_kept=0.8181 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=13.9s lr=2.000000e-03 loss=0.2353 loss_recon=0.2353 loss_meanflow=0.0000 mean_model_t=0.2075 mean_corrupt_t=0.2075 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3543 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9258 corrupt_frac=1.0000 acc_corrupt=0.9258 loss_corrupt=0.2353 wrong_frac=0.7924 init_acc_corrupt=0.1377 acc_corrupt_t_0p0_0p2=0.8669 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=12.4896 out_g_norm=0.2453 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3040 init_gold_top10=0.4599 init_gold_top100=0.5682 rollout_applied_pos_frac=0.3594 init_acc_rollout_applied=0.1761 init_acc_rollout_kept=0.1267 logit_acc_rollout_applied=0.8716 logit_acc_rollout_kept=0.9091 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt.log new file mode 100644 index 0000000000000000000000000000000000000000..cca21ef67a3e106be2c40e563f755ad458850a8f --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt.log @@ -0,0 +1,7919 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=50 epoch=50/500 epoch_step=1/1 micro_steps=50 elapsed=6.4s lr=2.000000e-03 loss=7.8264 loss_recon=7.8264 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0930 corrupt_frac=1.0000 acc_corrupt=0.0930 loss_corrupt=7.8264 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.0470 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.1256 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.2611 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.3951 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=0.4961 out_g_norm=0.8033 acc_corrupt_t_0p8_1p0=0.5176 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.7378 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=5.4s lr=2.000000e-03 loss=7.6153 loss_recon=7.6153 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0969 corrupt_frac=1.0000 acc_corrupt=0.0969 loss_corrupt=7.6153 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.0531 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.1274 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.2379 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=1.5226 out_g_norm=1.3656 acc_corrupt_t_0p6_0p8=0.3444 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.3613 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.4972 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=150 epoch=150/500 epoch_step=1/1 micro_steps=150 elapsed=5.4s lr=2.000000e-03 loss=7.2877 loss_recon=7.2877 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0994 corrupt_frac=1.0000 acc_corrupt=0.0994 loss_corrupt=7.2877 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.0542 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.1359 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.2490 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.3397 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=2.4311 out_g_norm=1.7252 acc_corrupt_t_0p8_1p0=0.4596 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.1431 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=5.4s lr=2.000000e-03 loss=6.8880 loss_recon=6.8880 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1070 corrupt_frac=1.0000 acc_corrupt=0.1070 loss_corrupt=6.8880 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.0570 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.1448 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.2597 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=3.3248 out_g_norm=1.8253 acc_corrupt_t_0p6_0p8=0.3654 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.4297 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.7784 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=250 epoch=250/500 epoch_step=1/1 micro_steps=250 elapsed=5.4s lr=2.000000e-03 loss=6.5803 loss_recon=6.5803 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1124 corrupt_frac=1.0000 acc_corrupt=0.1124 loss_corrupt=6.5803 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.0583 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.1512 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.2760 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.3882 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=4.1099 out_g_norm=1.5669 acc_corrupt_t_0p8_1p0=0.4849 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.4377 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=5.4s lr=2.000000e-03 loss=6.3095 loss_recon=6.3095 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1185 corrupt_frac=1.0000 acc_corrupt=0.1185 loss_corrupt=6.3095 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.0601 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.1594 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.2929 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.4075 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=4.6525 out_g_norm=1.0759 loss_all=6.1729 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=350 epoch=350/500 epoch_step=1/1 micro_steps=350 elapsed=5.4s lr=2.000000e-03 loss=6.0974 loss_recon=6.0974 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1202 corrupt_frac=1.0000 acc_corrupt=0.1202 loss_corrupt=6.0974 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.0628 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.1672 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.3055 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.4208 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=5.2130 out_g_norm=0.6219 loss_all=5.9160 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=5.4s lr=2.000000e-03 loss=5.8719 loss_recon=5.8719 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1275 corrupt_frac=1.0000 acc_corrupt=0.1275 loss_corrupt=5.8719 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.0643 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.1759 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.3252 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.4634 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=5.8672 out_g_norm=0.4203 loss_all=5.7728 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=450 epoch=450/500 epoch_step=1/1 micro_steps=450 elapsed=5.4s lr=2.000000e-03 loss=5.6177 loss_recon=5.6177 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1349 corrupt_frac=1.0000 acc_corrupt=0.1349 loss_corrupt=5.6177 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.0666 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.1844 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.3422 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=6.5218 out_g_norm=0.3793 acc_corrupt_t_0p6_0p8=0.4811 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.5879 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.5713 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=5.4s lr=2.000000e-03 loss=5.3320 loss_recon=5.3320 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1407 corrupt_frac=1.0000 acc_corrupt=0.1407 loss_corrupt=5.3320 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.0691 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.1950 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.3640 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.5252 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=7.1675 out_g_norm=0.3848 acc_corrupt_t_0p8_1p0=0.6855 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.1791 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=550 epoch=550/1000 epoch_step=1/1 micro_steps=550 elapsed=6.2s lr=2.000000e-03 loss=5.0272 loss_recon=5.0272 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1451 corrupt_frac=1.0000 acc_corrupt=0.1451 loss_corrupt=5.0272 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.0718 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.2045 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.3866 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.5443 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=7.8392 out_g_norm=0.4296 acc_corrupt_t_0p8_1p0=0.6689 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7271 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=5.4s lr=2.000000e-03 loss=4.6669 loss_recon=4.6669 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1584 corrupt_frac=1.0000 acc_corrupt=0.1584 loss_corrupt=4.6669 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.0741 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.2217 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.4106 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=8.5046 out_g_norm=0.4540 acc_corrupt_t_0p6_0p8=0.6006 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.7061 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.6568 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=650 epoch=650/1000 epoch_step=1/1 micro_steps=650 elapsed=5.4s lr=2.000000e-03 loss=4.3600 loss_recon=4.3600 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1679 corrupt_frac=1.0000 acc_corrupt=0.1679 loss_corrupt=4.3600 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.0768 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.2398 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.4770 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.6615 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.1193 out_g_norm=0.4495 acc_corrupt_t_0p8_1p0=0.8444 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.4069 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=5.4s lr=2.000000e-03 loss=4.0614 loss_recon=4.0614 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1846 corrupt_frac=1.0000 acc_corrupt=0.1846 loss_corrupt=4.0614 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.0812 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.2636 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.4995 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.6629 out_g_norm=0.5008 acc_corrupt_t_0p6_0p8=0.6812 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8447 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.1198 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=750 epoch=750/1000 epoch_step=1/1 micro_steps=750 elapsed=5.4s lr=2.000000e-03 loss=3.8332 loss_recon=3.8332 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1939 corrupt_frac=1.0000 acc_corrupt=0.1939 loss_corrupt=3.8332 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.0843 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.2778 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.5096 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.6826 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=10.1095 out_g_norm=0.5580 acc_corrupt_t_0p8_1p0=0.8145 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.7975 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=5.4s lr=2.000000e-03 loss=3.6357 loss_recon=3.6357 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2038 corrupt_frac=1.0000 acc_corrupt=0.2038 loss_corrupt=3.6357 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.0892 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.2918 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.5220 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.6905 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=10.4474 out_g_norm=0.6248 loss_all=3.5758 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=850 epoch=850/1000 epoch_step=1/1 micro_steps=850 elapsed=5.4s lr=2.000000e-03 loss=3.4888 loss_recon=3.4888 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2050 corrupt_frac=1.0000 acc_corrupt=0.2050 loss_corrupt=3.4888 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.0950 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.3043 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.5269 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.6866 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=10.7048 out_g_norm=0.7629 loss_all=3.2797 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=5.4s lr=2.000000e-03 loss=3.3242 loss_recon=3.3242 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2180 corrupt_frac=1.0000 acc_corrupt=0.2180 loss_corrupt=3.3242 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.1004 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.3199 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.5456 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.7094 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=10.9046 out_g_norm=0.8993 loss_all=3.2630 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=950 epoch=950/1000 epoch_step=1/1 micro_steps=950 elapsed=5.4s lr=2.000000e-03 loss=3.1398 loss_recon=3.1398 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2347 corrupt_frac=1.0000 acc_corrupt=0.2347 loss_corrupt=3.1398 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.1097 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.3405 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.5644 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.0699 out_g_norm=1.0239 acc_corrupt_t_0p6_0p8=0.7158 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.8701 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.1404 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=5.4s lr=2.000000e-03 loss=2.9363 loss_recon=2.9363 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2556 corrupt_frac=1.0000 acc_corrupt=0.2556 loss_corrupt=2.9363 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.1275 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.3723 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.5874 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.7488 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.2060 out_g_norm=1.0909 acc_corrupt_t_0p8_1p0=0.8623 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.8194 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1050 epoch=1050/1500 epoch_step=1/1 micro_steps=1050 elapsed=6.3s lr=2.000000e-03 loss=2.7334 loss_recon=2.7334 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2817 corrupt_frac=1.0000 acc_corrupt=0.2817 loss_corrupt=2.7334 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.1530 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.4085 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.6178 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.7593 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.3232 out_g_norm=1.2239 acc_corrupt_t_0p8_1p0=0.9023 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.4513 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=1100 epoch=1100/1500 epoch_step=1/1 micro_steps=1100 elapsed=5.5s lr=2.000000e-03 loss=2.4473 loss_recon=2.4473 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3314 corrupt_frac=1.0000 acc_corrupt=0.3314 loss_corrupt=2.4473 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.1918 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.4651 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.6478 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.4256 out_g_norm=1.4570 acc_corrupt_t_0p6_0p8=0.7773 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.8760 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.4251 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=1150 epoch=1150/1500 epoch_step=1/1 micro_steps=1150 elapsed=5.5s lr=2.000000e-03 loss=2.1991 loss_recon=2.1991 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3734 corrupt_frac=1.0000 acc_corrupt=0.3734 loss_corrupt=2.1991 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.2393 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.5154 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.6824 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.7879 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.5183 out_g_norm=1.6031 acc_corrupt_t_0p8_1p0=0.9010 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.2442 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=1200 epoch=1200/1500 epoch_step=1/1 micro_steps=1200 elapsed=5.5s lr=2.000000e-03 loss=1.9366 loss_recon=1.9366 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4193 corrupt_frac=1.0000 acc_corrupt=0.4193 loss_corrupt=1.9366 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.2808 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.5595 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7054 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6028 out_g_norm=1.5694 acc_corrupt_t_0p6_0p8=0.8111 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9170 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.9335 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=1250 epoch=1250/1500 epoch_step=1/1 micro_steps=1250 elapsed=5.5s lr=2.000000e-03 loss=1.7592 loss_recon=1.7592 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4469 corrupt_frac=1.0000 acc_corrupt=0.4469 loss_corrupt=1.7592 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.3085 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.5867 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7208 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8196 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6750 out_g_norm=1.6908 acc_corrupt_t_0p8_1p0=0.8965 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.7880 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=1300 epoch=1300/1500 epoch_step=1/1 micro_steps=1300 elapsed=5.5s lr=2.000000e-03 loss=1.6452 loss_recon=1.6452 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4632 corrupt_frac=1.0000 acc_corrupt=0.4632 loss_corrupt=1.6452 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.3246 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6027 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7321 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8233 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7165 out_g_norm=1.7272 loss_all=1.6747 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=1350 epoch=1350/1500 epoch_step=1/1 micro_steps=1350 elapsed=5.5s lr=2.000000e-03 loss=1.5839 loss_recon=1.5839 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4692 corrupt_frac=1.0000 acc_corrupt=0.4692 loss_corrupt=1.5839 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.3406 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6123 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7344 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8229 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7342 out_g_norm=1.6478 loss_all=1.4444 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=1400 epoch=1400/1500 epoch_step=1/1 micro_steps=1400 elapsed=5.5s lr=2.000000e-03 loss=1.5436 loss_recon=1.5436 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4751 corrupt_frac=1.0000 acc_corrupt=0.4751 loss_corrupt=1.5436 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.3423 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6177 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7391 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8330 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7304 out_g_norm=1.5622 loss_all=1.5482 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=1450 epoch=1450/1500 epoch_step=1/1 micro_steps=1450 elapsed=5.5s lr=2.000000e-03 loss=1.4775 loss_recon=1.4775 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4858 corrupt_frac=1.0000 acc_corrupt=0.4858 loss_corrupt=1.4775 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.3542 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6217 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7420 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7186 out_g_norm=1.5230 acc_corrupt_t_0p6_0p8=0.8334 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9248 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.6168 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=1500 epoch=1500/1500 epoch_step=1/1 micro_steps=1500 elapsed=5.5s lr=2.000000e-03 loss=1.4368 loss_recon=1.4368 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4908 corrupt_frac=1.0000 acc_corrupt=0.4908 loss_corrupt=1.4368 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.3640 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6255 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7433 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8423 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7144 out_g_norm=1.5139 acc_corrupt_t_0p8_1p0=0.9072 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4542 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1550 epoch=1550/2000 epoch_step=1/1 micro_steps=1550 elapsed=6.2s lr=2.000000e-03 loss=1.4253 loss_recon=1.4253 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4920 corrupt_frac=1.0000 acc_corrupt=0.4920 loss_corrupt=1.4253 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.3713 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6257 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7459 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8395 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7127 out_g_norm=1.5310 acc_corrupt_t_0p8_1p0=0.9258 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3091 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=5.4s lr=2.000000e-03 loss=1.3876 loss_recon=1.3876 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4989 corrupt_frac=1.0000 acc_corrupt=0.4989 loss_corrupt=1.3876 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.3724 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6303 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7440 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7011 out_g_norm=1.2570 acc_corrupt_t_0p6_0p8=0.8386 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9082 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4508 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=1650 epoch=1650/2000 epoch_step=1/1 micro_steps=1650 elapsed=5.4s lr=2.000000e-03 loss=1.3744 loss_recon=1.3744 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4990 corrupt_frac=1.0000 acc_corrupt=0.4990 loss_corrupt=1.3744 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.3805 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6298 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7489 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8362 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6924 out_g_norm=1.2805 acc_corrupt_t_0p8_1p0=0.9261 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4920 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=5.4s lr=2.000000e-03 loss=1.3508 loss_recon=1.3508 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5050 corrupt_frac=1.0000 acc_corrupt=0.5050 loss_corrupt=1.3508 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.3827 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6314 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7466 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6801 out_g_norm=1.2582 acc_corrupt_t_0p6_0p8=0.8406 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9307 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4714 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=1750 epoch=1750/2000 epoch_step=1/1 micro_steps=1750 elapsed=5.4s lr=2.000000e-03 loss=1.3325 loss_recon=1.3325 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5075 corrupt_frac=1.0000 acc_corrupt=0.5075 loss_corrupt=1.3325 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.3851 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6315 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7474 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8392 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6703 out_g_norm=1.1342 acc_corrupt_t_0p8_1p0=0.9092 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4953 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=5.4s lr=2.000000e-03 loss=1.3117 loss_recon=1.3117 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5106 corrupt_frac=1.0000 acc_corrupt=0.5106 loss_corrupt=1.3117 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.3881 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6331 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7501 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8386 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6645 out_g_norm=1.0733 loss_all=1.3930 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=1850 epoch=1850/2000 epoch_step=1/1 micro_steps=1850 elapsed=5.4s lr=2.000000e-03 loss=1.3172 loss_recon=1.3172 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5078 corrupt_frac=1.0000 acc_corrupt=0.5078 loss_corrupt=1.3172 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.3939 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6332 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7464 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8350 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6646 out_g_norm=1.1221 loss_all=1.2234 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=5.4s lr=2.000000e-03 loss=1.3210 loss_recon=1.3210 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5073 corrupt_frac=1.0000 acc_corrupt=0.5073 loss_corrupt=1.3210 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.3885 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6332 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7484 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8429 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6595 out_g_norm=1.0147 loss_all=1.2874 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=1950 epoch=1950/2000 epoch_step=1/1 micro_steps=1950 elapsed=5.4s lr=2.000000e-03 loss=1.2868 loss_recon=1.2868 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5145 corrupt_frac=1.0000 acc_corrupt=0.5145 loss_corrupt=1.2868 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.3971 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6336 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7505 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6586 out_g_norm=1.0138 acc_corrupt_t_0p6_0p8=0.8392 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9277 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4061 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=5.4s lr=2.000000e-03 loss=1.2733 loss_recon=1.2733 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5163 corrupt_frac=1.0000 acc_corrupt=0.5163 loss_corrupt=1.2733 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4030 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6344 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7496 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8495 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6619 out_g_norm=1.0338 acc_corrupt_t_0p8_1p0=0.9062 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2258 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2050 epoch=2050/2500 epoch_step=1/1 micro_steps=2050 elapsed=6.1s lr=2.000000e-03 loss=1.2651 loss_recon=1.2651 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5156 corrupt_frac=1.0000 acc_corrupt=0.5156 loss_corrupt=1.2651 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4070 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6344 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7503 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8450 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6691 out_g_norm=0.9582 acc_corrupt_t_0p8_1p0=0.9307 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1819 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=2100 epoch=2100/2500 epoch_step=1/1 micro_steps=2100 elapsed=5.4s lr=2.000000e-03 loss=1.2486 loss_recon=1.2486 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5225 corrupt_frac=1.0000 acc_corrupt=0.5225 loss_corrupt=1.2486 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4106 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6365 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7489 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6740 out_g_norm=0.9572 acc_corrupt_t_0p6_0p8=0.8429 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9092 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4426 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=2150 epoch=2150/2500 epoch_step=1/1 micro_steps=2150 elapsed=5.4s lr=2.000000e-03 loss=1.2684 loss_recon=1.2684 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5175 corrupt_frac=1.0000 acc_corrupt=0.5175 loss_corrupt=1.2684 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4092 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6351 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7527 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8397 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6715 out_g_norm=1.0153 acc_corrupt_t_0p8_1p0=0.9274 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4199 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=2200 epoch=2200/2500 epoch_step=1/1 micro_steps=2200 elapsed=5.4s lr=2.000000e-03 loss=1.2486 loss_recon=1.2486 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5226 corrupt_frac=1.0000 acc_corrupt=0.5226 loss_corrupt=1.2486 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4108 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6364 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7507 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6726 out_g_norm=0.9927 acc_corrupt_t_0p6_0p8=0.8456 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9414 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3096 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=2250 epoch=2250/2500 epoch_step=1/1 micro_steps=2250 elapsed=5.4s lr=2.000000e-03 loss=1.2227 loss_recon=1.2227 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5272 corrupt_frac=1.0000 acc_corrupt=0.5272 loss_corrupt=1.2227 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4174 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6360 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7516 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8435 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6696 out_g_norm=0.8981 acc_corrupt_t_0p8_1p0=0.9116 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3605 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=2300 epoch=2300/2500 epoch_step=1/1 micro_steps=2300 elapsed=5.4s lr=2.000000e-03 loss=1.2307 loss_recon=1.2307 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5270 corrupt_frac=1.0000 acc_corrupt=0.5270 loss_corrupt=1.2307 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4145 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6372 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7554 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8421 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6701 out_g_norm=0.8615 loss_all=1.3055 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=2350 epoch=2350/2500 epoch_step=1/1 micro_steps=2350 elapsed=5.4s lr=2.000000e-03 loss=1.2142 loss_recon=1.2142 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5258 corrupt_frac=1.0000 acc_corrupt=0.5258 loss_corrupt=1.2142 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4225 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6373 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7510 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8413 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6642 out_g_norm=0.8486 loss_all=1.1378 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=2400 epoch=2400/2500 epoch_step=1/1 micro_steps=2400 elapsed=5.4s lr=2.000000e-03 loss=1.2213 loss_recon=1.2213 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5277 corrupt_frac=1.0000 acc_corrupt=0.5277 loss_corrupt=1.2213 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4210 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6382 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7549 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8475 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6613 out_g_norm=0.8463 loss_all=1.1590 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=2450 epoch=2450/2500 epoch_step=1/1 micro_steps=2450 elapsed=5.4s lr=2.000000e-03 loss=1.2131 loss_recon=1.2131 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5290 corrupt_frac=1.0000 acc_corrupt=0.5290 loss_corrupt=1.2131 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4190 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6388 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7563 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6602 out_g_norm=0.8248 acc_corrupt_t_0p6_0p8=0.8473 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9307 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2607 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=2500 epoch=2500/2500 epoch_step=1/1 micro_steps=2500 elapsed=5.4s lr=2.000000e-03 loss=1.1799 loss_recon=1.1799 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5343 corrupt_frac=1.0000 acc_corrupt=0.5343 loss_corrupt=1.1799 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4306 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6401 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7569 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8569 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6669 out_g_norm=0.8173 acc_corrupt_t_0p8_1p0=0.9165 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1400 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2550 epoch=2550/3000 epoch_step=1/1 micro_steps=2550 elapsed=6.2s lr=2.000000e-03 loss=1.1954 loss_recon=1.1954 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5308 corrupt_frac=1.0000 acc_corrupt=0.5308 loss_corrupt=1.1954 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4305 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6381 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7580 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8519 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6735 out_g_norm=0.8356 acc_corrupt_t_0p8_1p0=0.9365 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0970 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=5.5s lr=2.000000e-03 loss=1.1644 loss_recon=1.1644 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5375 corrupt_frac=1.0000 acc_corrupt=0.5375 loss_corrupt=1.1644 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4333 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6416 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7558 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6860 out_g_norm=0.7980 acc_corrupt_t_0p6_0p8=0.8510 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9238 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2883 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=2650 epoch=2650/3000 epoch_step=1/1 micro_steps=2650 elapsed=5.6s lr=2.000000e-03 loss=1.1901 loss_recon=1.1901 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5314 corrupt_frac=1.0000 acc_corrupt=0.5314 loss_corrupt=1.1901 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4302 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6398 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7585 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8449 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6846 out_g_norm=0.8036 acc_corrupt_t_0p8_1p0=0.9316 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3064 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=5.5s lr=2.000000e-03 loss=1.1610 loss_recon=1.1610 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5385 corrupt_frac=1.0000 acc_corrupt=0.5385 loss_corrupt=1.1610 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4352 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6414 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7573 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6872 out_g_norm=0.6987 acc_corrupt_t_0p6_0p8=0.8508 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9355 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1981 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=2750 epoch=2750/3000 epoch_step=1/1 micro_steps=2750 elapsed=5.5s lr=2.000000e-03 loss=1.1537 loss_recon=1.1537 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5400 corrupt_frac=1.0000 acc_corrupt=0.5400 loss_corrupt=1.1537 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4370 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6402 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7577 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8503 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6895 out_g_norm=0.7700 acc_corrupt_t_0p8_1p0=0.9199 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2421 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=5.5s lr=2.000000e-03 loss=1.1432 loss_recon=1.1432 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5424 corrupt_frac=1.0000 acc_corrupt=0.5424 loss_corrupt=1.1432 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4393 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6412 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7603 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8484 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6835 out_g_norm=0.6981 loss_all=1.3854 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=2850 epoch=2850/3000 epoch_step=1/1 micro_steps=2850 elapsed=5.5s lr=2.000000e-03 loss=1.1433 loss_recon=1.1433 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5401 corrupt_frac=1.0000 acc_corrupt=0.5401 loss_corrupt=1.1433 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4446 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6413 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7573 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8475 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6858 out_g_norm=0.7106 loss_all=1.0435 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=5.4s lr=2.000000e-03 loss=1.1625 loss_recon=1.1625 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5377 corrupt_frac=1.0000 acc_corrupt=0.5377 loss_corrupt=1.1625 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4365 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6410 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7589 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8512 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6874 out_g_norm=0.8390 loss_all=1.1149 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=2950 epoch=2950/3000 epoch_step=1/1 micro_steps=2950 elapsed=5.4s lr=2.000000e-03 loss=1.1621 loss_recon=1.1621 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5392 corrupt_frac=1.0000 acc_corrupt=0.5392 loss_corrupt=1.1621 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4361 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6406 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7588 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6867 out_g_norm=0.8047 acc_corrupt_t_0p6_0p8=0.8488 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9336 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1794 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=5.4s lr=2.000000e-03 loss=1.1485 loss_recon=1.1485 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5405 corrupt_frac=1.0000 acc_corrupt=0.5405 loss_corrupt=1.1485 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4414 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6405 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7587 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8580 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6862 out_g_norm=0.7664 acc_corrupt_t_0p8_1p0=0.9180 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1026 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3050 epoch=3050/3500 epoch_step=1/1 micro_steps=3050 elapsed=6.3s lr=2.000000e-03 loss=1.1421 loss_recon=1.1421 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5407 corrupt_frac=1.0000 acc_corrupt=0.5407 loss_corrupt=1.1421 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4468 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6396 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7596 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8551 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6825 out_g_norm=0.6494 acc_corrupt_t_0p8_1p0=0.9229 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0287 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=3100 epoch=3100/3500 epoch_step=1/1 micro_steps=3100 elapsed=5.5s lr=2.000000e-03 loss=1.1316 loss_recon=1.1316 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5449 corrupt_frac=1.0000 acc_corrupt=0.5449 loss_corrupt=1.1316 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4454 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6431 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7575 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6940 out_g_norm=0.6834 acc_corrupt_t_0p6_0p8=0.8544 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9238 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2281 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=3150 epoch=3150/3500 epoch_step=1/1 micro_steps=3150 elapsed=5.5s lr=2.000000e-03 loss=1.1418 loss_recon=1.1418 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5408 corrupt_frac=1.0000 acc_corrupt=0.5408 loss_corrupt=1.1418 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4456 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6410 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7612 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8493 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6948 out_g_norm=0.6822 acc_corrupt_t_0p8_1p0=0.9342 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2501 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=3200 epoch=3200/3500 epoch_step=1/1 micro_steps=3200 elapsed=5.5s lr=2.000000e-03 loss=1.1174 loss_recon=1.1174 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5481 corrupt_frac=1.0000 acc_corrupt=0.5481 loss_corrupt=1.1174 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4518 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6423 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7601 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6920 out_g_norm=0.6215 acc_corrupt_t_0p6_0p8=0.8521 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9385 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1468 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=3250 epoch=3250/3500 epoch_step=1/1 micro_steps=3250 elapsed=5.5s lr=2.000000e-03 loss=1.1080 loss_recon=1.1080 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5486 corrupt_frac=1.0000 acc_corrupt=0.5486 loss_corrupt=1.1080 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4509 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6419 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7611 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8536 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6946 out_g_norm=0.6528 acc_corrupt_t_0p8_1p0=0.9150 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3192 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=3300 epoch=3300/3500 epoch_step=1/1 micro_steps=3300 elapsed=5.5s lr=2.000000e-03 loss=1.1053 loss_recon=1.1053 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5500 corrupt_frac=1.0000 acc_corrupt=0.5500 loss_corrupt=1.1053 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4518 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6423 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7631 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8517 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6948 out_g_norm=0.6681 loss_all=1.2291 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=3350 epoch=3350/3500 epoch_step=1/1 micro_steps=3350 elapsed=5.4s lr=2.000000e-03 loss=1.1014 loss_recon=1.1014 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5492 corrupt_frac=1.0000 acc_corrupt=0.5492 loss_corrupt=1.1014 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4598 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7591 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8472 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6957 out_g_norm=0.6499 loss_all=1.0135 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=3400 epoch=3400/3500 epoch_step=1/1 micro_steps=3400 elapsed=5.5s lr=2.000000e-03 loss=1.1110 loss_recon=1.1110 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5473 corrupt_frac=1.0000 acc_corrupt=0.5473 loss_corrupt=1.1110 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4520 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6429 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7619 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8561 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6970 out_g_norm=0.6560 loss_all=1.0370 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=3450 epoch=3450/3500 epoch_step=1/1 micro_steps=3450 elapsed=5.5s lr=2.000000e-03 loss=1.1110 loss_recon=1.1110 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5479 corrupt_frac=1.0000 acc_corrupt=0.5479 loss_corrupt=1.1110 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4499 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6427 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7621 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7053 out_g_norm=0.5879 acc_corrupt_t_0p6_0p8=0.8525 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9375 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1798 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=3500 epoch=3500/3500 epoch_step=1/1 micro_steps=3500 elapsed=5.4s lr=2.000000e-03 loss=1.0948 loss_recon=1.0948 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5506 corrupt_frac=1.0000 acc_corrupt=0.5506 loss_corrupt=1.0948 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4567 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6438 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7627 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8617 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7068 out_g_norm=0.6088 acc_corrupt_t_0p8_1p0=0.9180 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0952 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3550 epoch=3550/4000 epoch_step=1/1 micro_steps=3550 elapsed=6.2s lr=2.000000e-03 loss=1.0940 loss_recon=1.0940 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5489 corrupt_frac=1.0000 acc_corrupt=0.5489 loss_corrupt=1.0940 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4590 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6422 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7639 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8589 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7046 out_g_norm=0.6095 acc_corrupt_t_0p8_1p0=0.9385 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0479 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=5.5s lr=2.000000e-03 loss=1.0596 loss_recon=1.0596 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5576 corrupt_frac=1.0000 acc_corrupt=0.5576 loss_corrupt=1.0596 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4661 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6457 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7625 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7081 out_g_norm=0.5167 acc_corrupt_t_0p6_0p8=0.8583 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9199 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2026 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=3650 epoch=3650/4000 epoch_step=1/1 micro_steps=3650 elapsed=5.5s lr=2.000000e-03 loss=1.0873 loss_recon=1.0873 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5505 corrupt_frac=1.0000 acc_corrupt=0.5505 loss_corrupt=1.0873 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4598 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6447 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7658 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8553 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7149 out_g_norm=0.6524 acc_corrupt_t_0p8_1p0=0.9388 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1896 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=5.5s lr=2.000000e-03 loss=1.0704 loss_recon=1.0704 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5565 corrupt_frac=1.0000 acc_corrupt=0.5565 loss_corrupt=1.0704 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4646 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6449 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7632 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7104 out_g_norm=0.6117 acc_corrupt_t_0p6_0p8=0.8609 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9502 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0879 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=3750 epoch=3750/4000 epoch_step=1/1 micro_steps=3750 elapsed=5.4s lr=2.000000e-03 loss=1.0494 loss_recon=1.0494 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5598 corrupt_frac=1.0000 acc_corrupt=0.5598 loss_corrupt=1.0494 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4688 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6446 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7653 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8589 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7077 out_g_norm=0.6165 acc_corrupt_t_0p8_1p0=0.9180 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1485 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=5.4s lr=2.000000e-03 loss=1.0677 loss_recon=1.0677 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5565 corrupt_frac=1.0000 acc_corrupt=0.5565 loss_corrupt=1.0677 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4609 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6452 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7684 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8574 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7101 out_g_norm=0.6025 loss_all=1.0667 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=3850 epoch=3850/4000 epoch_step=1/1 micro_steps=3850 elapsed=5.5s lr=2.000000e-03 loss=1.0524 loss_recon=1.0524 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5569 corrupt_frac=1.0000 acc_corrupt=0.5569 loss_corrupt=1.0524 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4703 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6458 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7658 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8555 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7117 out_g_norm=0.5382 loss_all=0.9900 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=5.4s lr=2.000000e-03 loss=1.0589 loss_recon=1.0589 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5575 corrupt_frac=1.0000 acc_corrupt=0.5575 loss_corrupt=1.0589 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4673 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6459 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7692 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8641 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7166 out_g_norm=0.6598 loss_all=1.0573 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=3950 epoch=3950/4000 epoch_step=1/1 micro_steps=3950 elapsed=5.4s lr=2.000000e-03 loss=1.0856 loss_recon=1.0856 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5557 corrupt_frac=1.0000 acc_corrupt=0.5557 loss_corrupt=1.0856 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4610 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6455 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7694 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7156 out_g_norm=0.5969 acc_corrupt_t_0p6_0p8=0.8632 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9434 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1403 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=5.4s lr=2.000000e-03 loss=1.0691 loss_recon=1.0691 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5569 corrupt_frac=1.0000 acc_corrupt=0.5569 loss_corrupt=1.0691 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4651 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6467 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7701 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8709 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7082 out_g_norm=0.6061 acc_corrupt_t_0p8_1p0=0.9263 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1145 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4050 epoch=4050/4500 epoch_step=1/1 micro_steps=4050 elapsed=6.3s lr=2.000000e-03 loss=1.0569 loss_recon=1.0569 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5574 corrupt_frac=1.0000 acc_corrupt=0.5574 loss_corrupt=1.0569 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4702 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6459 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.7744 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.8709 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7060 out_g_norm=0.5900 acc_corrupt_t_0p8_1p0=0.9453 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9604 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=4100 epoch=4100/4500 epoch_step=1/1 micro_steps=4100 elapsed=5.6s lr=2.000000e-03 loss=1.0294 loss_recon=1.0294 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5647 corrupt_frac=1.0000 acc_corrupt=0.5647 loss_corrupt=1.0294 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4740 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6502 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.7747 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7124 out_g_norm=0.5563 acc_corrupt_t_0p6_0p8=0.8721 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9404 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1485 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=4150 epoch=4150/4500 epoch_step=1/1 micro_steps=4150 elapsed=5.5s lr=2.000000e-03 loss=1.0454 loss_recon=1.0454 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5600 corrupt_frac=1.0000 acc_corrupt=0.5600 loss_corrupt=1.0454 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4704 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6509 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.7831 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.8740 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7237 out_g_norm=0.6116 acc_corrupt_t_0p8_1p0=0.9535 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1227 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=4200 epoch=4200/4500 epoch_step=1/1 micro_steps=4200 elapsed=5.4s lr=2.000000e-03 loss=1.0254 loss_recon=1.0254 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5675 corrupt_frac=1.0000 acc_corrupt=0.5675 loss_corrupt=1.0254 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4769 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6523 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.7819 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7299 out_g_norm=0.5434 acc_corrupt_t_0p6_0p8=0.8797 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9551 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0414 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=4250 epoch=4250/4500 epoch_step=1/1 micro_steps=4250 elapsed=5.5s lr=2.000000e-03 loss=1.0175 loss_recon=1.0175 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5688 corrupt_frac=1.0000 acc_corrupt=0.5688 loss_corrupt=1.0175 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4754 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6541 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.7873 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.8830 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7312 out_g_norm=0.5893 acc_corrupt_t_0p8_1p0=0.9434 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1192 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=4300 epoch=4300/4500 epoch_step=1/1 micro_steps=4300 elapsed=5.5s lr=2.000000e-03 loss=1.0071 loss_recon=1.0071 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5714 corrupt_frac=1.0000 acc_corrupt=0.5714 loss_corrupt=1.0071 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4772 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6552 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.7939 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.8846 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7320 out_g_norm=0.5782 loss_all=1.0137 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=4350 epoch=4350/4500 epoch_step=1/1 micro_steps=4350 elapsed=5.5s lr=2.000000e-03 loss=1.0224 loss_recon=1.0224 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5682 corrupt_frac=1.0000 acc_corrupt=0.5682 loss_corrupt=1.0224 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4800 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6562 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.7915 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8851 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7396 out_g_norm=0.6082 loss_all=1.0064 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=4400 epoch=4400/4500 epoch_step=1/1 micro_steps=4400 elapsed=5.5s lr=2.000000e-03 loss=1.0396 loss_recon=1.0396 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5673 corrupt_frac=1.0000 acc_corrupt=0.5673 loss_corrupt=1.0396 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4742 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6563 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7957 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8941 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7389 out_g_norm=0.5622 loss_all=0.9332 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=4450 epoch=4450/4500 epoch_step=1/1 micro_steps=4450 elapsed=5.5s lr=2.000000e-03 loss=1.0234 loss_recon=1.0234 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5703 corrupt_frac=1.0000 acc_corrupt=0.5703 loss_corrupt=1.0234 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4736 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6592 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7998 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7380 out_g_norm=0.5097 acc_corrupt_t_0p6_0p8=0.8973 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9590 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1528 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=4500 epoch=4500/4500 epoch_step=1/1 micro_steps=4500 elapsed=5.5s lr=2.000000e-03 loss=1.0087 loss_recon=1.0087 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5723 corrupt_frac=1.0000 acc_corrupt=0.5723 loss_corrupt=1.0087 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4795 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6600 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8001 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9030 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7392 out_g_norm=0.5290 acc_corrupt_t_0p8_1p0=0.9526 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0055 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4550 epoch=4550/5000 epoch_step=1/1 micro_steps=4550 elapsed=6.2s lr=2.000000e-03 loss=1.0167 loss_recon=1.0167 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5693 corrupt_frac=1.0000 acc_corrupt=0.5693 loss_corrupt=1.0167 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4790 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6590 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8024 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9016 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7385 out_g_norm=0.5078 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9632 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=5.5s lr=2.000000e-03 loss=0.9899 loss_recon=0.9899 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5768 corrupt_frac=1.0000 acc_corrupt=0.5768 loss_corrupt=0.9899 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4829 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6635 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8017 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7401 out_g_norm=0.5749 acc_corrupt_t_0p6_0p8=0.9021 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9619 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1593 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=4650 epoch=4650/5000 epoch_step=1/1 micro_steps=4650 elapsed=5.5s lr=2.000000e-03 loss=1.0007 loss_recon=1.0007 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5726 corrupt_frac=1.0000 acc_corrupt=0.5726 loss_corrupt=1.0007 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4821 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6624 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8061 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9017 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7468 out_g_norm=0.5145 acc_corrupt_t_0p8_1p0=0.9723 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0252 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=5.5s lr=2.000000e-03 loss=0.9964 loss_recon=0.9964 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5767 corrupt_frac=1.0000 acc_corrupt=0.5767 loss_corrupt=0.9964 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4833 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6630 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8029 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7523 out_g_norm=0.5156 acc_corrupt_t_0p6_0p8=0.9042 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0656 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=4750 epoch=4750/5000 epoch_step=1/1 micro_steps=4750 elapsed=5.5s lr=2.000000e-03 loss=0.9723 loss_recon=0.9723 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5806 corrupt_frac=1.0000 acc_corrupt=0.5806 loss_corrupt=0.9723 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4876 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6638 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8054 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9038 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7533 out_g_norm=0.5221 acc_corrupt_t_0p8_1p0=0.9663 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0436 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=5.5s lr=2.000000e-03 loss=0.9928 loss_recon=0.9928 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5775 corrupt_frac=1.0000 acc_corrupt=0.5775 loss_corrupt=0.9928 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4806 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6630 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8086 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9005 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7528 out_g_norm=0.5318 loss_all=0.9960 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=4850 epoch=4850/5000 epoch_step=1/1 micro_steps=4850 elapsed=5.5s lr=2.000000e-03 loss=0.9898 loss_recon=0.9898 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5761 corrupt_frac=1.0000 acc_corrupt=0.5761 loss_corrupt=0.9898 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4872 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6642 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8043 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8991 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7512 out_g_norm=0.5502 loss_all=0.9272 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=5.5s lr=2.000000e-03 loss=1.0058 loss_recon=1.0058 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5747 corrupt_frac=1.0000 acc_corrupt=0.5747 loss_corrupt=1.0058 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4804 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6644 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8075 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9076 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7556 out_g_norm=0.5531 loss_all=0.9026 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=4950 epoch=4950/5000 epoch_step=1/1 micro_steps=4950 elapsed=5.5s lr=2.000000e-03 loss=1.0070 loss_recon=1.0070 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5762 corrupt_frac=1.0000 acc_corrupt=0.5762 loss_corrupt=1.0070 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4799 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6638 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8079 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7537 out_g_norm=0.4899 acc_corrupt_t_0p6_0p8=0.9055 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0581 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=5.6s lr=2.000000e-03 loss=0.9849 loss_recon=0.9849 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5778 corrupt_frac=1.0000 acc_corrupt=0.5778 loss_corrupt=0.9849 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4855 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6646 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8065 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7545 out_g_norm=0.4865 acc_corrupt_t_0p8_1p0=0.9600 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9465 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5050 epoch=5050/5500 epoch_step=1/1 micro_steps=5050 elapsed=6.2s lr=2.000000e-03 loss=0.9914 loss_recon=0.9914 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5752 corrupt_frac=1.0000 acc_corrupt=0.5752 loss_corrupt=0.9914 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4862 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6630 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8078 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9067 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7563 out_g_norm=0.5433 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9133 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=5100 epoch=5100/5500 epoch_step=1/1 micro_steps=5100 elapsed=5.5s lr=2.000000e-03 loss=0.9825 loss_recon=0.9825 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5787 corrupt_frac=1.0000 acc_corrupt=0.5787 loss_corrupt=0.9825 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4841 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6661 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8056 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7512 out_g_norm=0.5329 acc_corrupt_t_0p6_0p8=0.9071 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0208 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=5150 epoch=5150/5500 epoch_step=1/1 micro_steps=5150 elapsed=5.5s lr=2.000000e-03 loss=0.9672 loss_recon=0.9672 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5791 corrupt_frac=1.0000 acc_corrupt=0.5791 loss_corrupt=0.9672 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4900 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6670 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7540 out_g_norm=0.5172 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9574 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=5200 epoch=5200/5500 epoch_step=1/1 micro_steps=5200 elapsed=5.5s lr=2.000000e-03 loss=0.9575 loss_recon=0.9575 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5836 corrupt_frac=1.0000 acc_corrupt=0.5836 loss_corrupt=0.9575 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4917 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6679 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8078 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7581 out_g_norm=0.4500 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0537 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=5250 epoch=5250/5500 epoch_step=1/1 micro_steps=5250 elapsed=5.5s lr=2.000000e-03 loss=0.9799 loss_recon=0.9799 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5803 corrupt_frac=1.0000 acc_corrupt=0.5803 loss_corrupt=0.9799 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4855 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6655 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8078 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9063 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7527 out_g_norm=0.5506 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0177 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=5300 epoch=5300/5500 epoch_step=1/1 micro_steps=5300 elapsed=5.5s lr=2.000000e-03 loss=0.9702 loss_recon=0.9702 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5826 corrupt_frac=1.0000 acc_corrupt=0.5826 loss_corrupt=0.9702 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4875 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6660 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7460 out_g_norm=0.5336 loss_all=1.0178 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=5350 epoch=5350/5500 epoch_step=1/1 micro_steps=5350 elapsed=5.5s lr=2.000000e-03 loss=0.9665 loss_recon=0.9665 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5800 corrupt_frac=1.0000 acc_corrupt=0.5800 loss_corrupt=0.9665 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.4928 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6659 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8063 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9018 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7536 out_g_norm=0.5812 loss_all=0.9122 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=5400 epoch=5400/5500 epoch_step=1/1 micro_steps=5400 elapsed=5.4s lr=2.000000e-03 loss=0.9797 loss_recon=0.9797 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5794 corrupt_frac=1.0000 acc_corrupt=0.5794 loss_corrupt=0.9797 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4871 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6667 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8098 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7582 out_g_norm=0.4834 loss_all=0.8819 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=5450 epoch=5450/5500 epoch_step=1/1 micro_steps=5450 elapsed=5.5s lr=2.000000e-03 loss=0.9699 loss_recon=0.9699 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5816 corrupt_frac=1.0000 acc_corrupt=0.5816 loss_corrupt=0.9699 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4879 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6661 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8092 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7570 out_g_norm=0.4704 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1343 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=5500 epoch=5500/5500 epoch_step=1/1 micro_steps=5500 elapsed=5.5s lr=2.000000e-03 loss=0.9743 loss_recon=0.9743 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5796 corrupt_frac=1.0000 acc_corrupt=0.5796 loss_corrupt=0.9743 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4864 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6676 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8094 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7541 out_g_norm=0.5541 acc_corrupt_t_0p8_1p0=0.9614 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0850 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5550 epoch=5550/6000 epoch_step=1/1 micro_steps=5550 elapsed=6.6s lr=2.000000e-03 loss=0.9589 loss_recon=0.9589 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5814 corrupt_frac=1.0000 acc_corrupt=0.5814 loss_corrupt=0.9589 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4944 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6667 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7597 out_g_norm=0.4300 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8876 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=5.6s lr=2.000000e-03 loss=0.9338 loss_recon=0.9338 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5878 corrupt_frac=1.0000 acc_corrupt=0.5878 loss_corrupt=0.9338 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4972 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6705 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8085 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7665 out_g_norm=0.4471 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0225 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=5650 epoch=5650/6000 epoch_step=1/1 micro_steps=5650 elapsed=5.6s lr=2.000000e-03 loss=0.9588 loss_recon=0.9588 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5815 corrupt_frac=1.0000 acc_corrupt=0.5815 loss_corrupt=0.9588 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4933 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6683 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9075 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7687 out_g_norm=0.4901 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9829 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=5.5s lr=2.000000e-03 loss=0.9336 loss_recon=0.9336 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5886 corrupt_frac=1.0000 acc_corrupt=0.5886 loss_corrupt=0.9336 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4992 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6700 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8093 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7649 out_g_norm=0.4168 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0457 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=5750 epoch=5750/6000 epoch_step=1/1 micro_steps=5750 elapsed=5.5s lr=2.000000e-03 loss=0.9451 loss_recon=0.9451 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5867 corrupt_frac=1.0000 acc_corrupt=0.5867 loss_corrupt=0.9451 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4946 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6690 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8101 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7592 out_g_norm=0.4737 acc_corrupt_t_0p8_1p0=0.9683 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0060 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=5.6s lr=2.000000e-03 loss=0.9329 loss_recon=0.9329 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5884 corrupt_frac=1.0000 acc_corrupt=0.5884 loss_corrupt=0.9329 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4957 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6690 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9049 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7605 out_g_norm=0.4883 loss_all=0.9316 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=5850 epoch=5850/6000 epoch_step=1/1 micro_steps=5850 elapsed=5.5s lr=2.000000e-03 loss=0.9348 loss_recon=0.9348 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5861 corrupt_frac=1.0000 acc_corrupt=0.5861 loss_corrupt=0.9348 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5010 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6693 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8092 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9023 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7590 out_g_norm=0.4443 loss_all=0.8695 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=5.5s lr=2.000000e-03 loss=0.9416 loss_recon=0.9416 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5859 corrupt_frac=1.0000 acc_corrupt=0.5859 loss_corrupt=0.9416 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4964 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6698 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7571 out_g_norm=0.4177 loss_all=0.9402 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=5950 epoch=5950/6000 epoch_step=1/1 micro_steps=5950 elapsed=5.6s lr=2.000000e-03 loss=0.9450 loss_recon=0.9450 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5869 corrupt_frac=1.0000 acc_corrupt=0.5869 loss_corrupt=0.9450 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4957 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6687 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7457 out_g_norm=0.4808 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1437 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=5.5s lr=2.000000e-03 loss=0.9398 loss_recon=0.9398 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5858 corrupt_frac=1.0000 acc_corrupt=0.5858 loss_corrupt=0.9398 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4959 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6698 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9142 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7387 out_g_norm=0.4808 acc_corrupt_t_0p8_1p0=0.9600 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9987 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=6050 epoch=6050/6500 epoch_step=1/1 micro_steps=6050 elapsed=6.2s lr=2.000000e-03 loss=0.9395 loss_recon=0.9395 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5844 corrupt_frac=1.0000 acc_corrupt=0.5844 loss_corrupt=0.9395 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4991 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6675 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7337 out_g_norm=0.4699 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9256 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=6100 epoch=6100/6500 epoch_step=1/1 micro_steps=6100 elapsed=5.4s lr=2.000000e-03 loss=0.9303 loss_recon=0.9303 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5891 corrupt_frac=1.0000 acc_corrupt=0.5891 loss_corrupt=0.9303 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4993 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6711 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8085 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7268 out_g_norm=0.4738 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9768 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=6150 epoch=6150/6500 epoch_step=1/1 micro_steps=6150 elapsed=5.5s lr=2.000000e-03 loss=0.9699 loss_recon=0.9699 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5790 corrupt_frac=1.0000 acc_corrupt=0.5790 loss_corrupt=0.9699 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4891 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6681 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9075 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7245 out_g_norm=0.5582 acc_corrupt_t_0p8_1p0=0.9743 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0247 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=6200 epoch=6200/6500 epoch_step=1/1 micro_steps=6200 elapsed=5.4s lr=2.000000e-03 loss=0.9417 loss_recon=0.9417 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5870 corrupt_frac=1.0000 acc_corrupt=0.5870 loss_corrupt=0.9417 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4968 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6693 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8087 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7186 out_g_norm=0.4894 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0369 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=6250 epoch=6250/6500 epoch_step=1/1 micro_steps=6250 elapsed=5.5s lr=2.000000e-03 loss=0.9277 loss_recon=0.9277 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5906 corrupt_frac=1.0000 acc_corrupt=0.5906 loss_corrupt=0.9277 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5013 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6693 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7164 out_g_norm=0.5036 acc_corrupt_t_0p8_1p0=0.9722 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9545 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=6300 epoch=6300/6500 epoch_step=1/1 micro_steps=6300 elapsed=5.5s lr=2.000000e-03 loss=0.9306 loss_recon=0.9306 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5898 corrupt_frac=1.0000 acc_corrupt=0.5898 loss_corrupt=0.9306 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4981 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6696 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7163 out_g_norm=0.4946 loss_all=0.9168 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=6350 epoch=6350/6500 epoch_step=1/1 micro_steps=6350 elapsed=5.4s lr=2.000000e-03 loss=0.9316 loss_recon=0.9316 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5862 corrupt_frac=1.0000 acc_corrupt=0.5862 loss_corrupt=0.9316 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5010 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6697 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8088 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9039 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7101 out_g_norm=0.4848 loss_all=0.8718 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=6400 epoch=6400/6500 epoch_step=1/1 micro_steps=6400 elapsed=5.5s lr=2.000000e-03 loss=0.9377 loss_recon=0.9377 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5860 corrupt_frac=1.0000 acc_corrupt=0.5860 loss_corrupt=0.9377 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4964 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6701 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7108 out_g_norm=0.4415 loss_all=0.9103 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=6450 epoch=6450/6500 epoch_step=1/1 micro_steps=6450 elapsed=5.4s lr=2.000000e-03 loss=0.9339 loss_recon=0.9339 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5884 corrupt_frac=1.0000 acc_corrupt=0.5884 loss_corrupt=0.9339 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4974 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6701 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7184 out_g_norm=0.4694 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1209 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=6500 epoch=6500/6500 epoch_step=1/1 micro_steps=6500 elapsed=5.4s lr=2.000000e-03 loss=0.9175 loss_recon=0.9175 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5909 corrupt_frac=1.0000 acc_corrupt=0.5909 loss_corrupt=0.9175 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5034 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9143 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7238 out_g_norm=0.5007 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9797 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=6501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=6550 epoch=6550/7000 epoch_step=1/1 micro_steps=6550 elapsed=6.4s lr=2.000000e-03 loss=0.9323 loss_recon=0.9323 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5852 corrupt_frac=1.0000 acc_corrupt=0.5852 loss_corrupt=0.9323 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.4997 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6685 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7220 out_g_norm=0.4520 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8585 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=6600 epoch=6600/7000 epoch_step=1/1 micro_steps=6600 elapsed=5.5s lr=2.000000e-03 loss=0.9230 loss_recon=0.9230 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5893 corrupt_frac=1.0000 acc_corrupt=0.5893 loss_corrupt=0.9230 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.4994 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8093 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7214 out_g_norm=0.4700 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9678 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0257 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=6650 epoch=6650/7000 epoch_step=1/1 micro_steps=6650 elapsed=5.5s lr=2.000000e-03 loss=0.9536 loss_recon=0.9536 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5817 corrupt_frac=1.0000 acc_corrupt=0.5817 loss_corrupt=0.9536 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.4930 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6691 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7172 out_g_norm=0.5895 acc_corrupt_t_0p8_1p0=0.9743 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0051 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=6700 epoch=6700/7000 epoch_step=1/1 micro_steps=6700 elapsed=5.5s lr=2.000000e-03 loss=0.9184 loss_recon=0.9184 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5910 corrupt_frac=1.0000 acc_corrupt=0.5910 loss_corrupt=0.9184 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5030 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6707 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8099 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7143 out_g_norm=0.5135 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0355 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=6750 epoch=6750/7000 epoch_step=1/1 micro_steps=6750 elapsed=5.4s lr=2.000000e-03 loss=0.9191 loss_recon=0.9191 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5909 corrupt_frac=1.0000 acc_corrupt=0.5909 loss_corrupt=0.9191 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5011 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6704 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7183 out_g_norm=0.5230 acc_corrupt_t_0p8_1p0=0.9731 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0039 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=6800 epoch=6800/7000 epoch_step=1/1 micro_steps=6800 elapsed=5.5s lr=2.000000e-03 loss=0.9244 loss_recon=0.9244 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5902 corrupt_frac=1.0000 acc_corrupt=0.5902 loss_corrupt=0.9244 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.4987 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6697 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9066 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7209 out_g_norm=0.4653 loss_all=0.9160 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=6850 epoch=6850/7000 epoch_step=1/1 micro_steps=6850 elapsed=5.5s lr=2.000000e-03 loss=0.9311 loss_recon=0.9311 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5862 corrupt_frac=1.0000 acc_corrupt=0.5862 loss_corrupt=0.9311 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5007 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6699 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8094 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7208 out_g_norm=0.4746 loss_all=0.8891 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=6900 epoch=6900/7000 epoch_step=1/1 micro_steps=6900 elapsed=5.5s lr=2.000000e-03 loss=0.9313 loss_recon=0.9313 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5874 corrupt_frac=1.0000 acc_corrupt=0.5874 loss_corrupt=0.9313 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4988 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6702 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7156 out_g_norm=0.4361 loss_all=0.9449 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=6950 epoch=6950/7000 epoch_step=1/1 micro_steps=6950 elapsed=5.5s lr=2.000000e-03 loss=0.9254 loss_recon=0.9254 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5894 corrupt_frac=1.0000 acc_corrupt=0.5894 loss_corrupt=0.9254 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.4992 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6700 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7231 out_g_norm=0.4420 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9967 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=7000 epoch=7000/7000 epoch_step=1/1 micro_steps=7000 elapsed=5.5s lr=2.000000e-03 loss=0.9336 loss_recon=0.9336 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5882 corrupt_frac=1.0000 acc_corrupt=0.5882 loss_corrupt=0.9336 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.4992 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9153 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7270 out_g_norm=0.4711 acc_corrupt_t_0p8_1p0=0.9614 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9415 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=7001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=7050 epoch=7050/7500 epoch_step=1/1 micro_steps=7050 elapsed=6.2s lr=2.000000e-03 loss=0.9212 loss_recon=0.9212 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5882 corrupt_frac=1.0000 acc_corrupt=0.5882 loss_corrupt=0.9212 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5046 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6692 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7275 out_g_norm=0.4456 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8727 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=7100 epoch=7100/7500 epoch_step=1/1 micro_steps=7100 elapsed=5.4s lr=2.000000e-03 loss=0.9268 loss_recon=0.9268 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5899 corrupt_frac=1.0000 acc_corrupt=0.5899 loss_corrupt=0.9268 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5002 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8093 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7275 out_g_norm=0.4324 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9984 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=7150 epoch=7150/7500 epoch_step=1/1 micro_steps=7150 elapsed=5.5s lr=2.000000e-03 loss=0.9182 loss_recon=0.9182 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5878 corrupt_frac=1.0000 acc_corrupt=0.5878 loss_corrupt=0.9182 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5030 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6704 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7291 out_g_norm=0.4669 acc_corrupt_t_0p8_1p0=0.9759 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9853 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=7200 epoch=7200/7500 epoch_step=1/1 micro_steps=7200 elapsed=5.5s lr=2.000000e-03 loss=0.9061 loss_recon=0.9061 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5922 corrupt_frac=1.0000 acc_corrupt=0.5922 loss_corrupt=0.9061 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5047 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7291 out_g_norm=0.4681 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0290 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=7250 epoch=7250/7500 epoch_step=1/1 micro_steps=7250 elapsed=5.4s lr=2.000000e-03 loss=0.8966 loss_recon=0.8966 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5943 corrupt_frac=1.0000 acc_corrupt=0.5943 loss_corrupt=0.8966 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5065 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7300 out_g_norm=0.4596 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9725 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=7300 epoch=7300/7500 epoch_step=1/1 micro_steps=7300 elapsed=5.5s lr=2.000000e-03 loss=0.9090 loss_recon=0.9090 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5924 corrupt_frac=1.0000 acc_corrupt=0.5924 loss_corrupt=0.9090 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5013 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6716 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8143 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9064 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7325 out_g_norm=0.4417 loss_all=0.9161 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=7350 epoch=7350/7500 epoch_step=1/1 micro_steps=7350 elapsed=5.5s lr=2.000000e-03 loss=0.9181 loss_recon=0.9181 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5884 corrupt_frac=1.0000 acc_corrupt=0.5884 loss_corrupt=0.9181 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5041 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6708 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8098 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9053 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7323 out_g_norm=0.4629 loss_all=0.8870 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=7400 epoch=7400/7500 epoch_step=1/1 micro_steps=7400 elapsed=5.5s lr=2.000000e-03 loss=0.9124 loss_recon=0.9124 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5907 corrupt_frac=1.0000 acc_corrupt=0.5907 loss_corrupt=0.9124 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5037 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7362 out_g_norm=0.4106 loss_all=0.8427 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=7450 epoch=7450/7500 epoch_step=1/1 micro_steps=7450 elapsed=5.5s lr=2.000000e-03 loss=0.8946 loss_recon=0.8946 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5949 corrupt_frac=1.0000 acc_corrupt=0.5949 loss_corrupt=0.8946 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5079 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6719 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7449 out_g_norm=0.4187 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0399 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=7500 epoch=7500/7500 epoch_step=1/1 micro_steps=7500 elapsed=5.4s lr=2.000000e-03 loss=0.9190 loss_recon=0.9190 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5900 corrupt_frac=1.0000 acc_corrupt=0.5900 loss_corrupt=0.9190 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5017 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7418 out_g_norm=0.4899 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8936 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=7501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=7550 epoch=7550/8000 epoch_step=1/1 micro_steps=7550 elapsed=6.3s lr=2.000000e-03 loss=0.9243 loss_recon=0.9243 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5868 corrupt_frac=1.0000 acc_corrupt=0.5868 loss_corrupt=0.9243 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5024 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6687 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7334 out_g_norm=0.5104 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8922 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=7600 epoch=7600/8000 epoch_step=1/1 micro_steps=7600 elapsed=5.5s lr=2.000000e-03 loss=0.9201 loss_recon=0.9201 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5906 corrupt_frac=1.0000 acc_corrupt=0.5906 loss_corrupt=0.9201 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5011 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8092 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7294 out_g_norm=0.4190 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9540 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=7650 epoch=7650/8000 epoch_step=1/1 micro_steps=7650 elapsed=5.5s lr=2.000000e-03 loss=0.9185 loss_recon=0.9185 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5888 corrupt_frac=1.0000 acc_corrupt=0.5888 loss_corrupt=0.9185 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5041 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7312 out_g_norm=0.4211 acc_corrupt_t_0p8_1p0=0.9749 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0159 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=7700 epoch=7700/8000 epoch_step=1/1 micro_steps=7700 elapsed=5.5s lr=2.000000e-03 loss=0.8965 loss_recon=0.8965 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5943 corrupt_frac=1.0000 acc_corrupt=0.5943 loss_corrupt=0.8965 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5078 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7372 out_g_norm=0.4879 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9314 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=7750 epoch=7750/8000 epoch_step=1/1 micro_steps=7750 elapsed=5.5s lr=2.000000e-03 loss=0.9132 loss_recon=0.9132 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5915 corrupt_frac=1.0000 acc_corrupt=0.5915 loss_corrupt=0.9132 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5018 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6710 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7393 out_g_norm=0.4634 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8936 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=7800 epoch=7800/8000 epoch_step=1/1 micro_steps=7800 elapsed=5.5s lr=2.000000e-03 loss=0.8960 loss_recon=0.8960 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5954 corrupt_frac=1.0000 acc_corrupt=0.5954 loss_corrupt=0.8960 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5063 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9067 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7459 out_g_norm=0.4024 loss_all=0.9700 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=7850 epoch=7850/8000 epoch_step=1/1 micro_steps=7850 elapsed=5.5s lr=2.000000e-03 loss=0.9027 loss_recon=0.9027 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5910 corrupt_frac=1.0000 acc_corrupt=0.5910 loss_corrupt=0.9027 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5071 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7597 out_g_norm=0.3768 loss_all=0.8772 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=7900 epoch=7900/8000 epoch_step=1/1 micro_steps=7900 elapsed=5.5s lr=2.000000e-03 loss=0.9221 loss_recon=0.9221 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5888 corrupt_frac=1.0000 acc_corrupt=0.5888 loss_corrupt=0.9221 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5006 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6713 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7640 out_g_norm=0.3857 loss_all=0.9137 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=7950 epoch=7950/8000 epoch_step=1/1 micro_steps=7950 elapsed=5.5s lr=2.000000e-03 loss=0.9070 loss_recon=0.9070 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5931 corrupt_frac=1.0000 acc_corrupt=0.5931 loss_corrupt=0.9070 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5048 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6717 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7600 out_g_norm=0.4400 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1292 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=8000 epoch=8000/8000 epoch_step=1/1 micro_steps=8000 elapsed=5.5s lr=2.000000e-03 loss=0.8955 loss_recon=0.8955 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5934 corrupt_frac=1.0000 acc_corrupt=0.5934 loss_corrupt=0.8955 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5073 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7553 out_g_norm=0.4055 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9228 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=8001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=8050 epoch=8050/8500 epoch_step=1/1 micro_steps=8050 elapsed=6.2s lr=2.000000e-03 loss=0.9151 loss_recon=0.9151 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5883 corrupt_frac=1.0000 acc_corrupt=0.5883 loss_corrupt=0.9151 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5044 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6698 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7554 out_g_norm=0.4524 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9006 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=8100 epoch=8100/8500 epoch_step=1/1 micro_steps=8100 elapsed=5.5s lr=2.000000e-03 loss=0.9000 loss_recon=0.9000 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5941 corrupt_frac=1.0000 acc_corrupt=0.5941 loss_corrupt=0.9000 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5067 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8099 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7529 out_g_norm=0.4519 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9992 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=8150 epoch=8150/8500 epoch_step=1/1 micro_steps=8150 elapsed=5.5s lr=2.000000e-03 loss=0.9087 loss_recon=0.9087 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5894 corrupt_frac=1.0000 acc_corrupt=0.5894 loss_corrupt=0.9087 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5047 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6719 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9082 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7546 out_g_norm=0.3897 acc_corrupt_t_0p8_1p0=0.9743 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0227 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=8200 epoch=8200/8500 epoch_step=1/1 micro_steps=8200 elapsed=5.5s lr=2.000000e-03 loss=0.9057 loss_recon=0.9057 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5926 corrupt_frac=1.0000 acc_corrupt=0.5926 loss_corrupt=0.9057 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5048 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7479 out_g_norm=0.4280 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0045 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=8250 epoch=8250/8500 epoch_step=1/1 micro_steps=8250 elapsed=5.5s lr=2.000000e-03 loss=0.9098 loss_recon=0.9098 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5926 corrupt_frac=1.0000 acc_corrupt=0.5926 loss_corrupt=0.9098 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5037 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6710 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9089 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7395 out_g_norm=0.4337 acc_corrupt_t_0p8_1p0=0.9546 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0866 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=8300 epoch=8300/8500 epoch_step=1/1 micro_steps=8300 elapsed=5.5s lr=2.000000e-03 loss=0.8874 loss_recon=0.8874 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5970 corrupt_frac=1.0000 acc_corrupt=0.5970 loss_corrupt=0.8874 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5093 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8148 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9077 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7414 out_g_norm=0.4201 loss_all=0.9101 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=8350 epoch=8350/8500 epoch_step=1/1 micro_steps=8350 elapsed=5.5s lr=2.000000e-03 loss=0.9062 loss_recon=0.9062 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5909 corrupt_frac=1.0000 acc_corrupt=0.5909 loss_corrupt=0.9062 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5077 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6719 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8101 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9038 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7526 out_g_norm=0.3886 loss_all=0.8625 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=8400 epoch=8400/8500 epoch_step=1/1 micro_steps=8400 elapsed=5.5s lr=2.000000e-03 loss=0.9014 loss_recon=0.9014 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5916 corrupt_frac=1.0000 acc_corrupt=0.5916 loss_corrupt=0.9014 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5052 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6718 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7595 out_g_norm=0.3915 loss_all=0.8574 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=8450 epoch=8450/8500 epoch_step=1/1 micro_steps=8450 elapsed=5.5s lr=2.000000e-03 loss=0.9135 loss_recon=0.9135 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5914 corrupt_frac=1.0000 acc_corrupt=0.5914 loss_corrupt=0.9135 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5021 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7544 out_g_norm=0.4555 acc_corrupt_t_0p6_0p8=0.9089 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9853 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=8500 epoch=8500/8500 epoch_step=1/1 micro_steps=8500 elapsed=5.5s lr=2.000000e-03 loss=0.9018 loss_recon=0.9018 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5929 corrupt_frac=1.0000 acc_corrupt=0.5929 loss_corrupt=0.9018 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5068 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9156 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7493 out_g_norm=0.4183 acc_corrupt_t_0p8_1p0=0.9624 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0243 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=8501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=8550 epoch=8550/9000 epoch_step=1/1 micro_steps=8550 elapsed=6.3s lr=2.000000e-03 loss=0.9080 loss_recon=0.9080 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5894 corrupt_frac=1.0000 acc_corrupt=0.5894 loss_corrupt=0.9080 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5064 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6697 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7497 out_g_norm=0.4162 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8453 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=8600 epoch=8600/9000 epoch_step=1/1 micro_steps=8600 elapsed=5.5s lr=2.000000e-03 loss=0.8881 loss_recon=0.8881 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5959 corrupt_frac=1.0000 acc_corrupt=0.5959 loss_corrupt=0.8881 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5100 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8098 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7486 out_g_norm=0.4002 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9801 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=8650 epoch=8650/9000 epoch_step=1/1 micro_steps=8650 elapsed=5.5s lr=2.000000e-03 loss=0.9063 loss_recon=0.9063 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5891 corrupt_frac=1.0000 acc_corrupt=0.5891 loss_corrupt=0.9063 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5040 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6723 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7558 out_g_norm=0.4077 acc_corrupt_t_0p8_1p0=0.9772 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9735 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=8700 epoch=8700/9000 epoch_step=1/1 micro_steps=8700 elapsed=5.5s lr=2.000000e-03 loss=0.8931 loss_recon=0.8931 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5950 corrupt_frac=1.0000 acc_corrupt=0.5950 loss_corrupt=0.8931 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5089 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7540 out_g_norm=0.4848 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9967 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=8750 epoch=8750/9000 epoch_step=1/1 micro_steps=8750 elapsed=5.5s lr=2.000000e-03 loss=0.9133 loss_recon=0.9133 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5921 corrupt_frac=1.0000 acc_corrupt=0.5921 loss_corrupt=0.9133 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5028 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6709 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7529 out_g_norm=0.4705 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0877 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=8800 epoch=8800/9000 epoch_step=1/1 micro_steps=8800 elapsed=5.5s lr=2.000000e-03 loss=0.9102 loss_recon=0.9102 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5936 corrupt_frac=1.0000 acc_corrupt=0.5936 loss_corrupt=0.9102 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5035 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9068 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7531 out_g_norm=0.4403 loss_all=0.8928 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=8850 epoch=8850/9000 epoch_step=1/1 micro_steps=8850 elapsed=5.5s lr=2.000000e-03 loss=0.9046 loss_recon=0.9046 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5908 corrupt_frac=1.0000 acc_corrupt=0.5908 loss_corrupt=0.9046 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5073 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9049 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7556 out_g_norm=0.4351 loss_all=0.8747 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=8900 epoch=8900/9000 epoch_step=1/1 micro_steps=8900 elapsed=5.5s lr=2.000000e-03 loss=0.9095 loss_recon=0.9095 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5903 corrupt_frac=1.0000 acc_corrupt=0.5903 loss_corrupt=0.9095 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5030 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6716 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7605 out_g_norm=0.4326 loss_all=0.8751 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=8950 epoch=8950/9000 epoch_step=1/1 micro_steps=8950 elapsed=5.5s lr=2.000000e-03 loss=0.9028 loss_recon=0.9028 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5929 corrupt_frac=1.0000 acc_corrupt=0.5929 loss_corrupt=0.9028 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5044 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6714 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7596 out_g_norm=0.4189 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0230 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=9000 epoch=9000/9000 epoch_step=1/1 micro_steps=9000 elapsed=5.5s lr=2.000000e-03 loss=0.9096 loss_recon=0.9096 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5914 corrupt_frac=1.0000 acc_corrupt=0.5914 loss_corrupt=0.9096 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5050 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6711 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9148 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7526 out_g_norm=0.4534 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9206 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=9001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=9050 epoch=9050/9500 epoch_step=1/1 micro_steps=9050 elapsed=6.3s lr=2.000000e-03 loss=0.9051 loss_recon=0.9051 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5900 corrupt_frac=1.0000 acc_corrupt=0.5900 loss_corrupt=0.9051 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5073 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6700 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7522 out_g_norm=0.4238 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8613 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=9100 epoch=9100/9500 epoch_step=1/1 micro_steps=9100 elapsed=5.5s lr=2.000000e-03 loss=0.8875 loss_recon=0.8875 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5954 corrupt_frac=1.0000 acc_corrupt=0.5954 loss_corrupt=0.8875 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5093 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8099 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7655 out_g_norm=0.4051 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9110 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=9150 epoch=9150/9500 epoch_step=1/1 micro_steps=9150 elapsed=5.4s lr=2.000000e-03 loss=0.9056 loss_recon=0.9056 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5910 corrupt_frac=1.0000 acc_corrupt=0.5910 loss_corrupt=0.9056 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5072 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7743 out_g_norm=0.4380 acc_corrupt_t_0p8_1p0=0.9759 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9771 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=9200 epoch=9200/9500 epoch_step=1/1 micro_steps=9200 elapsed=5.4s lr=2.000000e-03 loss=0.9006 loss_recon=0.9006 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5931 corrupt_frac=1.0000 acc_corrupt=0.5931 loss_corrupt=0.9006 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5060 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7675 out_g_norm=0.4232 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9678 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=9250 epoch=9250/9500 epoch_step=1/1 micro_steps=9250 elapsed=5.4s lr=2.000000e-03 loss=0.9050 loss_recon=0.9050 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5925 corrupt_frac=1.0000 acc_corrupt=0.5925 loss_corrupt=0.9050 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5034 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6713 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7596 out_g_norm=0.4368 acc_corrupt_t_0p8_1p0=0.9712 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0056 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=9300 epoch=9300/9500 epoch_step=1/1 micro_steps=9300 elapsed=5.4s lr=2.000000e-03 loss=0.8868 loss_recon=0.8868 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5960 corrupt_frac=1.0000 acc_corrupt=0.5960 loss_corrupt=0.8868 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5078 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6716 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8143 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9075 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7626 out_g_norm=0.4535 loss_all=0.8371 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=9350 epoch=9350/9500 epoch_step=1/1 micro_steps=9350 elapsed=5.4s lr=2.000000e-03 loss=0.9099 loss_recon=0.9099 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5892 corrupt_frac=1.0000 acc_corrupt=0.5892 loss_corrupt=0.9099 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5048 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6718 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9043 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7597 out_g_norm=0.3934 loss_all=0.8997 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=9400 epoch=9400/9500 epoch_step=1/1 micro_steps=9400 elapsed=5.4s lr=2.000000e-03 loss=0.9037 loss_recon=0.9037 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5917 corrupt_frac=1.0000 acc_corrupt=0.5917 loss_corrupt=0.9037 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5051 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7611 out_g_norm=0.4346 loss_all=0.8589 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=9450 epoch=9450/9500 epoch_step=1/1 micro_steps=9450 elapsed=5.4s lr=2.000000e-03 loss=0.8928 loss_recon=0.8928 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5949 corrupt_frac=1.0000 acc_corrupt=0.5949 loss_corrupt=0.8928 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5078 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7704 out_g_norm=0.4093 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0140 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=9500 epoch=9500/9500 epoch_step=1/1 micro_steps=9500 elapsed=5.4s lr=2.000000e-03 loss=0.8945 loss_recon=0.8945 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5932 corrupt_frac=1.0000 acc_corrupt=0.5932 loss_corrupt=0.8945 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5070 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9146 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7722 out_g_norm=0.4019 acc_corrupt_t_0p8_1p0=0.9634 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9821 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=9501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=9550 epoch=9550/10000 epoch_step=1/1 micro_steps=9550 elapsed=6.2s lr=2.000000e-03 loss=0.8963 loss_recon=0.8963 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5912 corrupt_frac=1.0000 acc_corrupt=0.5912 loss_corrupt=0.8963 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5091 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6704 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7688 out_g_norm=0.3736 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8731 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=9600 epoch=9600/10000 epoch_step=1/1 micro_steps=9600 elapsed=5.5s lr=2.000000e-03 loss=0.8814 loss_recon=0.8814 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8814 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5116 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7728 out_g_norm=0.4114 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9345 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=9650 epoch=9650/10000 epoch_step=1/1 micro_steps=9650 elapsed=5.4s lr=2.000000e-03 loss=0.8932 loss_recon=0.8932 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5924 corrupt_frac=1.0000 acc_corrupt=0.5924 loss_corrupt=0.8932 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5096 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7797 out_g_norm=0.3797 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0226 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=9700 epoch=9700/10000 epoch_step=1/1 micro_steps=9700 elapsed=5.5s lr=2.000000e-03 loss=0.8858 loss_recon=0.8858 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5953 corrupt_frac=1.0000 acc_corrupt=0.5953 loss_corrupt=0.8858 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5090 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7785 out_g_norm=0.3842 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9481 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=9750 epoch=9750/10000 epoch_step=1/1 micro_steps=9750 elapsed=5.5s lr=2.000000e-03 loss=0.8955 loss_recon=0.8955 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5941 corrupt_frac=1.0000 acc_corrupt=0.5941 loss_corrupt=0.8955 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5062 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7730 out_g_norm=0.4014 acc_corrupt_t_0p8_1p0=0.9712 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9275 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=9800 epoch=9800/10000 epoch_step=1/1 micro_steps=9800 elapsed=5.5s lr=2.000000e-03 loss=0.8856 loss_recon=0.8856 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5965 corrupt_frac=1.0000 acc_corrupt=0.5965 loss_corrupt=0.8856 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5083 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9064 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7704 out_g_norm=0.4313 loss_all=0.8471 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=9850 epoch=9850/10000 epoch_step=1/1 micro_steps=9850 elapsed=5.5s lr=2.000000e-03 loss=0.8905 loss_recon=0.8905 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5929 corrupt_frac=1.0000 acc_corrupt=0.5929 loss_corrupt=0.8905 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5109 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8099 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9050 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7655 out_g_norm=0.4163 loss_all=0.9045 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=9900 epoch=9900/10000 epoch_step=1/1 micro_steps=9900 elapsed=5.5s lr=2.000000e-03 loss=0.8839 loss_recon=0.8839 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5952 corrupt_frac=1.0000 acc_corrupt=0.5952 loss_corrupt=0.8839 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5107 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9124 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7691 out_g_norm=0.4146 loss_all=0.8830 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=9950 epoch=9950/10000 epoch_step=1/1 micro_steps=9950 elapsed=5.5s lr=2.000000e-03 loss=0.8770 loss_recon=0.8770 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5975 corrupt_frac=1.0000 acc_corrupt=0.5975 loss_corrupt=0.8770 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5120 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6725 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7793 out_g_norm=0.4630 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9989 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=10000 epoch=10000/10000 epoch_step=1/1 micro_steps=10000 elapsed=5.4s lr=2.000000e-03 loss=0.8916 loss_recon=0.8916 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5943 corrupt_frac=1.0000 acc_corrupt=0.5943 loss_corrupt=0.8916 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5089 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6730 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9158 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7721 out_g_norm=0.4040 acc_corrupt_t_0p8_1p0=0.9624 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9098 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=10001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=10050 epoch=10050/10500 epoch_step=1/1 micro_steps=10050 elapsed=6.3s lr=2.000000e-03 loss=0.8806 loss_recon=0.8806 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5942 corrupt_frac=1.0000 acc_corrupt=0.5942 loss_corrupt=0.8806 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5138 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6713 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7664 out_g_norm=0.3679 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8772 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=10100 epoch=10100/10500 epoch_step=1/1 micro_steps=10100 elapsed=5.5s lr=2.000000e-03 loss=0.8767 loss_recon=0.8767 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8767 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5123 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7752 out_g_norm=0.4027 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9580 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9997 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=10150 epoch=10150/10500 epoch_step=1/1 micro_steps=10150 elapsed=5.5s lr=2.000000e-03 loss=0.9141 loss_recon=0.9141 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5890 corrupt_frac=1.0000 acc_corrupt=0.5890 loss_corrupt=0.9141 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5036 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7717 out_g_norm=0.4049 acc_corrupt_t_0p8_1p0=0.9753 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9902 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=10200 epoch=10200/10500 epoch_step=1/1 micro_steps=10200 elapsed=5.5s lr=2.000000e-03 loss=0.8773 loss_recon=0.8773 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5971 corrupt_frac=1.0000 acc_corrupt=0.5971 loss_corrupt=0.8773 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5122 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7665 out_g_norm=0.3646 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9458 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=10250 epoch=10250/10500 epoch_step=1/1 micro_steps=10250 elapsed=5.5s lr=2.000000e-03 loss=0.8816 loss_recon=0.8816 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5962 corrupt_frac=1.0000 acc_corrupt=0.5962 loss_corrupt=0.8816 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5089 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7804 out_g_norm=0.3510 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9252 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=10300 epoch=10300/10500 epoch_step=1/1 micro_steps=10300 elapsed=5.4s lr=2.000000e-03 loss=0.8810 loss_recon=0.8810 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5979 corrupt_frac=1.0000 acc_corrupt=0.5979 loss_corrupt=0.8810 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5101 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9073 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7796 out_g_norm=0.3694 loss_all=0.8383 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=10350 epoch=10350/10500 epoch_step=1/1 micro_steps=10350 elapsed=5.4s lr=2.000000e-03 loss=0.8732 loss_recon=0.8732 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5953 corrupt_frac=1.0000 acc_corrupt=0.5953 loss_corrupt=0.8732 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5145 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9047 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7738 out_g_norm=0.3495 loss_all=0.8244 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=10400 epoch=10400/10500 epoch_step=1/1 micro_steps=10400 elapsed=5.4s lr=2.000000e-03 loss=0.8898 loss_recon=0.8898 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5933 corrupt_frac=1.0000 acc_corrupt=0.5933 loss_corrupt=0.8898 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5074 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7747 out_g_norm=0.3992 loss_all=0.8585 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=10450 epoch=10450/10500 epoch_step=1/1 micro_steps=10450 elapsed=5.4s lr=2.000000e-03 loss=0.8964 loss_recon=0.8964 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5941 corrupt_frac=1.0000 acc_corrupt=0.5941 loss_corrupt=0.8964 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5060 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7757 out_g_norm=0.4617 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1348 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=10500 epoch=10500/10500 epoch_step=1/1 micro_steps=10500 elapsed=5.4s lr=2.000000e-03 loss=0.8946 loss_recon=0.8946 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5933 corrupt_frac=1.0000 acc_corrupt=0.5933 loss_corrupt=0.8946 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5077 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9137 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7657 out_g_norm=0.4355 acc_corrupt_t_0p8_1p0=0.9590 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9134 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=10501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=10550 epoch=10550/11000 epoch_step=1/1 micro_steps=10550 elapsed=6.3s lr=2.000000e-03 loss=0.8846 loss_recon=0.8846 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5928 corrupt_frac=1.0000 acc_corrupt=0.5928 loss_corrupt=0.8846 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5113 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6714 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7704 out_g_norm=0.3656 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8808 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=10600 epoch=10600/11000 epoch_step=1/1 micro_steps=10600 elapsed=5.6s lr=2.000000e-03 loss=0.8861 loss_recon=0.8861 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5957 corrupt_frac=1.0000 acc_corrupt=0.5957 loss_corrupt=0.8861 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5096 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7697 out_g_norm=0.4075 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8802 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=10650 epoch=10650/11000 epoch_step=1/1 micro_steps=10650 elapsed=5.6s lr=2.000000e-03 loss=0.8841 loss_recon=0.8841 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5919 corrupt_frac=1.0000 acc_corrupt=0.5919 loss_corrupt=0.8841 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5089 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7663 out_g_norm=0.3887 acc_corrupt_t_0p8_1p0=0.9753 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9830 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=10700 epoch=10700/11000 epoch_step=1/1 micro_steps=10700 elapsed=5.6s lr=2.000000e-03 loss=0.8734 loss_recon=0.8734 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5982 corrupt_frac=1.0000 acc_corrupt=0.5982 loss_corrupt=0.8734 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5136 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7736 out_g_norm=0.3685 acc_corrupt_t_0p6_0p8=0.9115 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9447 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=10750 epoch=10750/11000 epoch_step=1/1 micro_steps=10750 elapsed=5.6s lr=2.000000e-03 loss=0.8899 loss_recon=0.8899 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5956 corrupt_frac=1.0000 acc_corrupt=0.5956 loss_corrupt=0.8899 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5085 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9087 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7728 out_g_norm=0.4059 acc_corrupt_t_0p8_1p0=0.9683 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9034 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=10800 epoch=10800/11000 epoch_step=1/1 micro_steps=10800 elapsed=5.6s lr=2.000000e-03 loss=0.8850 loss_recon=0.8850 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5970 corrupt_frac=1.0000 acc_corrupt=0.5970 loss_corrupt=0.8850 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5086 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6730 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9069 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7793 out_g_norm=0.3882 loss_all=0.8306 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=10850 epoch=10850/11000 epoch_step=1/1 micro_steps=10850 elapsed=5.6s lr=2.000000e-03 loss=0.8858 loss_recon=0.8858 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5939 corrupt_frac=1.0000 acc_corrupt=0.5939 loss_corrupt=0.8858 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5124 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8098 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9053 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7740 out_g_norm=0.3831 loss_all=0.8273 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=10900 epoch=10900/11000 epoch_step=1/1 micro_steps=10900 elapsed=5.6s lr=2.000000e-03 loss=0.8978 loss_recon=0.8978 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5918 corrupt_frac=1.0000 acc_corrupt=0.5918 loss_corrupt=0.8978 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5051 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6723 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7746 out_g_norm=0.4318 loss_all=0.8353 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=10950 epoch=10950/11000 epoch_step=1/1 micro_steps=10950 elapsed=5.7s lr=2.000000e-03 loss=0.8961 loss_recon=0.8961 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5925 corrupt_frac=1.0000 acc_corrupt=0.5925 loss_corrupt=0.8961 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5038 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7750 out_g_norm=0.4107 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0111 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=11000 epoch=11000/11000 epoch_step=1/1 micro_steps=11000 elapsed=5.5s lr=2.000000e-03 loss=0.8816 loss_recon=0.8816 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5957 corrupt_frac=1.0000 acc_corrupt=0.5957 loss_corrupt=0.8816 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5115 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9154 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7746 out_g_norm=0.3741 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8882 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=11001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=11050 epoch=11050/11500 epoch_step=1/1 micro_steps=11050 elapsed=6.3s lr=2.000000e-03 loss=0.8782 loss_recon=0.8782 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5933 corrupt_frac=1.0000 acc_corrupt=0.5933 loss_corrupt=0.8782 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6708 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7798 out_g_norm=0.3833 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8031 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=11100 epoch=11100/11500 epoch_step=1/1 micro_steps=11100 elapsed=5.4s lr=2.000000e-03 loss=0.8700 loss_recon=0.8700 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5985 corrupt_frac=1.0000 acc_corrupt=0.5985 loss_corrupt=0.8700 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5136 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7875 out_g_norm=0.3396 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8791 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=11150 epoch=11150/11500 epoch_step=1/1 micro_steps=11150 elapsed=5.5s lr=2.000000e-03 loss=0.8759 loss_recon=0.8759 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5944 corrupt_frac=1.0000 acc_corrupt=0.5944 loss_corrupt=0.8759 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5130 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7900 out_g_norm=0.3812 acc_corrupt_t_0p8_1p0=0.9759 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9168 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=11200 epoch=11200/11500 epoch_step=1/1 micro_steps=11200 elapsed=5.4s lr=2.000000e-03 loss=0.8797 loss_recon=0.8797 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5973 corrupt_frac=1.0000 acc_corrupt=0.5973 loss_corrupt=0.8797 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5121 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7913 out_g_norm=0.4555 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8936 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=11250 epoch=11250/11500 epoch_step=1/1 micro_steps=11250 elapsed=5.4s lr=2.000000e-03 loss=0.8877 loss_recon=0.8877 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5953 corrupt_frac=1.0000 acc_corrupt=0.5953 loss_corrupt=0.8877 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5079 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6719 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7837 out_g_norm=0.3768 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9071 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=11300 epoch=11300/11500 epoch_step=1/1 micro_steps=11300 elapsed=5.4s lr=2.000000e-03 loss=0.8785 loss_recon=0.8785 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8785 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5096 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6730 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8149 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9077 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7883 out_g_norm=0.3961 loss_all=0.8500 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=11350 epoch=11350/11500 epoch_step=1/1 micro_steps=11350 elapsed=5.5s lr=2.000000e-03 loss=0.8833 loss_recon=0.8833 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5937 corrupt_frac=1.0000 acc_corrupt=0.5937 loss_corrupt=0.8833 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5121 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9059 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7860 out_g_norm=0.4435 loss_all=0.8244 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=11400 epoch=11400/11500 epoch_step=1/1 micro_steps=11400 elapsed=5.5s lr=2.000000e-03 loss=0.8849 loss_recon=0.8849 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5953 corrupt_frac=1.0000 acc_corrupt=0.5953 loss_corrupt=0.8849 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5109 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7866 out_g_norm=0.3636 loss_all=0.7914 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=11450 epoch=11450/11500 epoch_step=1/1 micro_steps=11450 elapsed=5.5s lr=2.000000e-03 loss=0.8679 loss_recon=0.8679 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5982 corrupt_frac=1.0000 acc_corrupt=0.5982 loss_corrupt=0.8679 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5129 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7963 out_g_norm=0.3968 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0394 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=11500 epoch=11500/11500 epoch_step=1/1 micro_steps=11500 elapsed=5.6s lr=2.000000e-03 loss=0.8826 loss_recon=0.8826 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5959 corrupt_frac=1.0000 acc_corrupt=0.5959 loss_corrupt=0.8826 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5109 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9163 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7872 out_g_norm=0.3595 acc_corrupt_t_0p8_1p0=0.9619 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9286 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=11501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=11550 epoch=11550/12000 epoch_step=1/1 micro_steps=11550 elapsed=6.2s lr=2.000000e-03 loss=0.8808 loss_recon=0.8808 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5931 corrupt_frac=1.0000 acc_corrupt=0.5931 loss_corrupt=0.8808 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5119 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7822 out_g_norm=0.3868 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8048 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=11600 epoch=11600/12000 epoch_step=1/1 micro_steps=11600 elapsed=5.4s lr=2.000000e-03 loss=0.8780 loss_recon=0.8780 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8780 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5118 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7831 out_g_norm=0.4106 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9492 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8919 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=11650 epoch=11650/12000 epoch_step=1/1 micro_steps=11650 elapsed=5.7s lr=2.000000e-03 loss=0.8821 loss_recon=0.8821 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5934 corrupt_frac=1.0000 acc_corrupt=0.5934 loss_corrupt=0.8821 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5113 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9080 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7854 out_g_norm=0.4312 acc_corrupt_t_0p8_1p0=0.9743 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9219 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=11700 epoch=11700/12000 epoch_step=1/1 micro_steps=11700 elapsed=5.8s lr=2.000000e-03 loss=0.8628 loss_recon=0.8628 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5992 corrupt_frac=1.0000 acc_corrupt=0.5992 loss_corrupt=0.8628 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5152 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7920 out_g_norm=0.3703 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8719 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=11750 epoch=11750/12000 epoch_step=1/1 micro_steps=11750 elapsed=5.8s lr=2.000000e-03 loss=0.8779 loss_recon=0.8779 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8779 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5117 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7943 out_g_norm=0.3969 acc_corrupt_t_0p8_1p0=0.9712 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8666 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=11800 epoch=11800/12000 epoch_step=1/1 micro_steps=11800 elapsed=5.6s lr=2.000000e-03 loss=0.8670 loss_recon=0.8670 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6002 corrupt_frac=1.0000 acc_corrupt=0.6002 loss_corrupt=0.8670 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5143 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8146 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9065 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7884 out_g_norm=0.3748 loss_all=0.8588 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=11850 epoch=11850/12000 epoch_step=1/1 micro_steps=11850 elapsed=5.6s lr=2.000000e-03 loss=0.8732 loss_recon=0.8732 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5951 corrupt_frac=1.0000 acc_corrupt=0.5951 loss_corrupt=0.8732 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5143 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9059 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7927 out_g_norm=0.4228 loss_all=0.7941 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=11900 epoch=11900/12000 epoch_step=1/1 micro_steps=11900 elapsed=5.5s lr=2.000000e-03 loss=0.8719 loss_recon=0.8719 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5968 corrupt_frac=1.0000 acc_corrupt=0.5968 loss_corrupt=0.8719 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5133 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8064 out_g_norm=0.3965 loss_all=0.8130 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=11950 epoch=11950/12000 epoch_step=1/1 micro_steps=11950 elapsed=5.5s lr=2.000000e-03 loss=0.8853 loss_recon=0.8853 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5960 corrupt_frac=1.0000 acc_corrupt=0.5960 loss_corrupt=0.8853 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5095 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6725 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8091 out_g_norm=0.4276 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9851 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=12000 epoch=12000/12000 epoch_step=1/1 micro_steps=12000 elapsed=5.5s lr=2.000000e-03 loss=0.8697 loss_recon=0.8697 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5971 corrupt_frac=1.0000 acc_corrupt=0.5971 loss_corrupt=0.8697 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5132 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9147 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8102 out_g_norm=0.4037 acc_corrupt_t_0p8_1p0=0.9609 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9949 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=12001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=12050 epoch=12050/12500 epoch_step=1/1 micro_steps=12050 elapsed=6.2s lr=2.000000e-03 loss=0.8775 loss_recon=0.8775 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5946 corrupt_frac=1.0000 acc_corrupt=0.5946 loss_corrupt=0.8775 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5141 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6719 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8068 out_g_norm=0.3801 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8462 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=12100 epoch=12100/12500 epoch_step=1/1 micro_steps=12100 elapsed=5.4s lr=2.000000e-03 loss=0.8807 loss_recon=0.8807 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5971 corrupt_frac=1.0000 acc_corrupt=0.5971 loss_corrupt=0.8807 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5109 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8045 out_g_norm=0.3846 acc_corrupt_t_0p6_0p8=0.9115 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0290 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=12150 epoch=12150/12500 epoch_step=1/1 micro_steps=12150 elapsed=5.6s lr=2.000000e-03 loss=0.8816 loss_recon=0.8816 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5937 corrupt_frac=1.0000 acc_corrupt=0.5937 loss_corrupt=0.8816 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5118 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7998 out_g_norm=0.3988 acc_corrupt_t_0p8_1p0=0.9762 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8550 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=12200 epoch=12200/12500 epoch_step=1/1 micro_steps=12200 elapsed=5.5s lr=2.000000e-03 loss=0.8647 loss_recon=0.8647 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5995 corrupt_frac=1.0000 acc_corrupt=0.5995 loss_corrupt=0.8647 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5160 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8024 out_g_norm=0.3840 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9053 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=12250 epoch=12250/12500 epoch_step=1/1 micro_steps=12250 elapsed=5.5s lr=2.000000e-03 loss=0.8706 loss_recon=0.8706 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5988 corrupt_frac=1.0000 acc_corrupt=0.5988 loss_corrupt=0.8706 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5129 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8064 out_g_norm=0.3492 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9538 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=12300 epoch=12300/12500 epoch_step=1/1 micro_steps=12300 elapsed=5.5s lr=2.000000e-03 loss=0.8677 loss_recon=0.8677 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5998 corrupt_frac=1.0000 acc_corrupt=0.5998 loss_corrupt=0.8677 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5128 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9070 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8086 out_g_norm=0.3656 loss_all=0.9017 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=12350 epoch=12350/12500 epoch_step=1/1 micro_steps=12350 elapsed=5.5s lr=2.000000e-03 loss=0.8703 loss_recon=0.8703 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5956 corrupt_frac=1.0000 acc_corrupt=0.5956 loss_corrupt=0.8703 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5149 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8099 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9064 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8003 out_g_norm=0.3995 loss_all=0.8243 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=12400 epoch=12400/12500 epoch_step=1/1 micro_steps=12400 elapsed=5.6s lr=2.000000e-03 loss=0.8719 loss_recon=0.8719 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5963 corrupt_frac=1.0000 acc_corrupt=0.5963 loss_corrupt=0.8719 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7930 out_g_norm=0.3883 loss_all=0.8469 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=12450 epoch=12450/12500 epoch_step=1/1 micro_steps=12450 elapsed=5.6s lr=2.000000e-03 loss=0.8824 loss_recon=0.8824 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5953 corrupt_frac=1.0000 acc_corrupt=0.5953 loss_corrupt=0.8824 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5076 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7923 out_g_norm=0.3932 acc_corrupt_t_0p6_0p8=0.9087 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9778 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=12500 epoch=12500/12500 epoch_step=1/1 micro_steps=12500 elapsed=5.5s lr=2.000000e-03 loss=0.8610 loss_recon=0.8610 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5990 corrupt_frac=1.0000 acc_corrupt=0.5990 loss_corrupt=0.8610 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5165 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9148 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7869 out_g_norm=0.3844 acc_corrupt_t_0p8_1p0=0.9585 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9050 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=12501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=12550 epoch=12550/13000 epoch_step=1/1 micro_steps=12550 elapsed=6.3s lr=2.000000e-03 loss=0.8709 loss_recon=0.8709 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5942 corrupt_frac=1.0000 acc_corrupt=0.5942 loss_corrupt=0.8709 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5133 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7887 out_g_norm=0.3511 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8329 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=12600 epoch=12600/13000 epoch_step=1/1 micro_steps=12600 elapsed=5.5s lr=2.000000e-03 loss=0.8713 loss_recon=0.8713 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5979 corrupt_frac=1.0000 acc_corrupt=0.5979 loss_corrupt=0.8713 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5128 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7886 out_g_norm=0.4188 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9677 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=12650 epoch=12650/13000 epoch_step=1/1 micro_steps=12650 elapsed=5.5s lr=2.000000e-03 loss=0.8740 loss_recon=0.8740 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5941 corrupt_frac=1.0000 acc_corrupt=0.5941 loss_corrupt=0.8740 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5118 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7812 out_g_norm=0.3515 acc_corrupt_t_0p8_1p0=0.9769 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8882 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=12700 epoch=12700/13000 epoch_step=1/1 micro_steps=12700 elapsed=5.5s lr=2.000000e-03 loss=0.8659 loss_recon=0.8659 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5987 corrupt_frac=1.0000 acc_corrupt=0.5987 loss_corrupt=0.8659 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5147 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7808 out_g_norm=0.4314 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8766 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=12750 epoch=12750/13000 epoch_step=1/1 micro_steps=12750 elapsed=5.5s lr=2.000000e-03 loss=0.8758 loss_recon=0.8758 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5974 corrupt_frac=1.0000 acc_corrupt=0.5974 loss_corrupt=0.8758 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5115 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7808 out_g_norm=0.3798 acc_corrupt_t_0p8_1p0=0.9492 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9050 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=12800 epoch=12800/13000 epoch_step=1/1 micro_steps=12800 elapsed=5.5s lr=2.000000e-03 loss=0.8634 loss_recon=0.8634 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6001 corrupt_frac=1.0000 acc_corrupt=0.6001 loss_corrupt=0.8634 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5141 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9073 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7786 out_g_norm=0.3680 loss_all=0.8571 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=12850 epoch=12850/13000 epoch_step=1/1 micro_steps=12850 elapsed=5.5s lr=2.000000e-03 loss=0.8786 loss_recon=0.8786 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5944 corrupt_frac=1.0000 acc_corrupt=0.5944 loss_corrupt=0.8786 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5131 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9066 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7777 out_g_norm=0.3725 loss_all=0.8730 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=12900 epoch=12900/13000 epoch_step=1/1 micro_steps=12900 elapsed=5.5s lr=2.000000e-03 loss=0.8662 loss_recon=0.8662 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5974 corrupt_frac=1.0000 acc_corrupt=0.5974 loss_corrupt=0.8662 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5147 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7762 out_g_norm=0.3474 loss_all=0.7857 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=12950 epoch=12950/13000 epoch_step=1/1 micro_steps=12950 elapsed=5.5s lr=2.000000e-03 loss=0.8632 loss_recon=0.8632 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5985 corrupt_frac=1.0000 acc_corrupt=0.5985 loss_corrupt=0.8632 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5134 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7817 out_g_norm=0.3404 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8984 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=13000 epoch=13000/13000 epoch_step=1/1 micro_steps=13000 elapsed=5.5s lr=2.000000e-03 loss=0.8662 loss_recon=0.8662 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5980 corrupt_frac=1.0000 acc_corrupt=0.5980 loss_corrupt=0.8662 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5149 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9150 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7845 out_g_norm=0.3561 acc_corrupt_t_0p8_1p0=0.9648 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9197 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=13001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=13050 epoch=13050/13500 epoch_step=1/1 micro_steps=13050 elapsed=6.2s lr=2.000000e-03 loss=0.8823 loss_recon=0.8823 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5933 corrupt_frac=1.0000 acc_corrupt=0.5933 loss_corrupt=0.8823 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6710 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7782 out_g_norm=0.3733 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8026 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=13100 epoch=13100/13500 epoch_step=1/1 micro_steps=13100 elapsed=5.4s lr=2.000000e-03 loss=0.8712 loss_recon=0.8712 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5981 corrupt_frac=1.0000 acc_corrupt=0.5981 loss_corrupt=0.8712 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5130 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7784 out_g_norm=0.3892 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9483 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=13150 epoch=13150/13500 epoch_step=1/1 micro_steps=13150 elapsed=5.4s lr=2.000000e-03 loss=0.8933 loss_recon=0.8933 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5914 corrupt_frac=1.0000 acc_corrupt=0.5914 loss_corrupt=0.8933 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5082 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9080 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7701 out_g_norm=0.4118 acc_corrupt_t_0p8_1p0=0.9762 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9109 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=13200 epoch=13200/13500 epoch_step=1/1 micro_steps=13200 elapsed=5.5s lr=2.000000e-03 loss=0.8796 loss_recon=0.8796 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5967 corrupt_frac=1.0000 acc_corrupt=0.5967 loss_corrupt=0.8796 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5114 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7672 out_g_norm=0.4265 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8963 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=13250 epoch=13250/13500 epoch_step=1/1 micro_steps=13250 elapsed=5.6s lr=2.000000e-03 loss=0.8622 loss_recon=0.8622 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5997 corrupt_frac=1.0000 acc_corrupt=0.5997 loss_corrupt=0.8622 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5153 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9084 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7682 out_g_norm=0.3944 acc_corrupt_t_0p8_1p0=0.9692 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9735 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=13300 epoch=13300/13500 epoch_step=1/1 micro_steps=13300 elapsed=5.6s lr=2.000000e-03 loss=0.8540 loss_recon=0.8540 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6020 corrupt_frac=1.0000 acc_corrupt=0.6020 loss_corrupt=0.8540 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5174 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9058 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7721 out_g_norm=0.3599 loss_all=0.8021 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=13350 epoch=13350/13500 epoch_step=1/1 micro_steps=13350 elapsed=5.4s lr=2.000000e-03 loss=0.8693 loss_recon=0.8693 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5960 corrupt_frac=1.0000 acc_corrupt=0.5960 loss_corrupt=0.8693 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5155 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8103 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9060 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7784 out_g_norm=0.4174 loss_all=0.7957 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=13400 epoch=13400/13500 epoch_step=1/1 micro_steps=13400 elapsed=5.4s lr=2.000000e-03 loss=0.8611 loss_recon=0.8611 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5987 corrupt_frac=1.0000 acc_corrupt=0.5987 loss_corrupt=0.8611 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5164 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7804 out_g_norm=0.3626 loss_all=0.8152 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=13450 epoch=13450/13500 epoch_step=1/1 micro_steps=13450 elapsed=5.4s lr=2.000000e-03 loss=0.8773 loss_recon=0.8773 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5977 corrupt_frac=1.0000 acc_corrupt=0.5977 loss_corrupt=0.8773 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5124 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6725 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7780 out_g_norm=0.3583 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9948 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=13500 epoch=13500/13500 epoch_step=1/1 micro_steps=13500 elapsed=5.4s lr=2.000000e-03 loss=0.8793 loss_recon=0.8793 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5961 corrupt_frac=1.0000 acc_corrupt=0.5961 loss_corrupt=0.8793 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5114 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6740 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9140 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7820 out_g_norm=0.3764 acc_corrupt_t_0p8_1p0=0.9614 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8635 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=13501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=13550 epoch=13550/14000 epoch_step=1/1 micro_steps=13550 elapsed=6.2s lr=2.000000e-03 loss=0.8788 loss_recon=0.8788 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5940 corrupt_frac=1.0000 acc_corrupt=0.5940 loss_corrupt=0.8788 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5134 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6714 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7792 out_g_norm=0.3622 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8480 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=13600 epoch=13600/14000 epoch_step=1/1 micro_steps=13600 elapsed=5.4s lr=2.000000e-03 loss=0.8591 loss_recon=0.8591 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5998 corrupt_frac=1.0000 acc_corrupt=0.5998 loss_corrupt=0.8591 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5160 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7796 out_g_norm=0.3353 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9678 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8860 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=13650 epoch=13650/14000 epoch_step=1/1 micro_steps=13650 elapsed=5.5s lr=2.000000e-03 loss=0.8744 loss_recon=0.8744 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5946 corrupt_frac=1.0000 acc_corrupt=0.5946 loss_corrupt=0.8744 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5133 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7774 out_g_norm=0.3748 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9596 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=13700 epoch=13700/14000 epoch_step=1/1 micro_steps=13700 elapsed=5.5s lr=2.000000e-03 loss=0.8665 loss_recon=0.8665 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5979 corrupt_frac=1.0000 acc_corrupt=0.5979 loss_corrupt=0.8665 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5140 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7686 out_g_norm=0.3988 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8838 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=13750 epoch=13750/14000 epoch_step=1/1 micro_steps=13750 elapsed=5.5s lr=2.000000e-03 loss=0.8584 loss_recon=0.8584 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6009 corrupt_frac=1.0000 acc_corrupt=0.6009 loss_corrupt=0.8584 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5173 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7700 out_g_norm=0.3845 acc_corrupt_t_0p8_1p0=0.9561 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9113 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=13800 epoch=13800/14000 epoch_step=1/1 micro_steps=13800 elapsed=5.4s lr=2.000000e-03 loss=0.8748 loss_recon=0.8748 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5975 corrupt_frac=1.0000 acc_corrupt=0.5975 loss_corrupt=0.8748 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5098 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6725 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9074 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7745 out_g_norm=0.4030 loss_all=0.8366 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=13850 epoch=13850/14000 epoch_step=1/1 micro_steps=13850 elapsed=5.4s lr=2.000000e-03 loss=0.8747 loss_recon=0.8747 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5949 corrupt_frac=1.0000 acc_corrupt=0.5949 loss_corrupt=0.8747 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5140 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7748 out_g_norm=0.3996 loss_all=0.7685 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=13900 epoch=13900/14000 epoch_step=1/1 micro_steps=13900 elapsed=5.5s lr=2.000000e-03 loss=0.8444 loss_recon=0.8444 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6013 corrupt_frac=1.0000 acc_corrupt=0.6013 loss_corrupt=0.8444 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5207 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9132 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7884 out_g_norm=0.3242 loss_all=0.8328 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=13950 epoch=13950/14000 epoch_step=1/1 micro_steps=13950 elapsed=5.5s lr=2.000000e-03 loss=0.8827 loss_recon=0.8827 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5957 corrupt_frac=1.0000 acc_corrupt=0.5957 loss_corrupt=0.8827 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5084 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6730 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7978 out_g_norm=0.3608 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0352 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=14000 epoch=14000/14000 epoch_step=1/1 micro_steps=14000 elapsed=5.5s lr=2.000000e-03 loss=0.8666 loss_recon=0.8666 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5986 corrupt_frac=1.0000 acc_corrupt=0.5986 loss_corrupt=0.8666 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5158 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9153 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7947 out_g_norm=0.3891 acc_corrupt_t_0p8_1p0=0.9570 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8949 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=14001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=14050 epoch=14050/14500 epoch_step=1/1 micro_steps=14050 elapsed=6.3s lr=2.000000e-03 loss=0.8776 loss_recon=0.8776 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5940 corrupt_frac=1.0000 acc_corrupt=0.5940 loss_corrupt=0.8776 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5134 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7914 out_g_norm=0.3681 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8365 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=14100 epoch=14100/14500 epoch_step=1/1 micro_steps=14100 elapsed=5.6s lr=2.000000e-03 loss=0.8725 loss_recon=0.8725 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8725 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6740 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8103 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7884 out_g_norm=0.3629 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9259 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=14150 epoch=14150/14500 epoch_step=1/1 micro_steps=14150 elapsed=5.6s lr=2.000000e-03 loss=0.8655 loss_recon=0.8655 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5957 corrupt_frac=1.0000 acc_corrupt=0.5957 loss_corrupt=0.8655 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5149 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8146 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7968 out_g_norm=0.3847 acc_corrupt_t_0p8_1p0=0.9759 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8575 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=14200 epoch=14200/14500 epoch_step=1/1 micro_steps=14200 elapsed=5.6s lr=2.000000e-03 loss=0.8524 loss_recon=0.8524 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6009 corrupt_frac=1.0000 acc_corrupt=0.6009 loss_corrupt=0.8524 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5181 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7974 out_g_norm=0.3544 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9304 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=14250 epoch=14250/14500 epoch_step=1/1 micro_steps=14250 elapsed=5.5s lr=2.000000e-03 loss=0.8761 loss_recon=0.8761 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5977 corrupt_frac=1.0000 acc_corrupt=0.5977 loss_corrupt=0.8761 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5112 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7939 out_g_norm=0.3650 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9157 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=14300 epoch=14300/14500 epoch_step=1/1 micro_steps=14300 elapsed=5.5s lr=2.000000e-03 loss=0.8462 loss_recon=0.8462 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6023 corrupt_frac=1.0000 acc_corrupt=0.6023 loss_corrupt=0.8462 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5177 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8152 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9073 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7947 out_g_norm=0.3242 loss_all=0.8015 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=14350 epoch=14350/14500 epoch_step=1/1 micro_steps=14350 elapsed=5.5s lr=2.000000e-03 loss=0.8610 loss_recon=0.8610 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5967 corrupt_frac=1.0000 acc_corrupt=0.5967 loss_corrupt=0.8610 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5167 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8009 out_g_norm=0.3387 loss_all=0.8514 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=14400 epoch=14400/14500 epoch_step=1/1 micro_steps=14400 elapsed=5.6s lr=2.000000e-03 loss=0.8767 loss_recon=0.8767 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5951 corrupt_frac=1.0000 acc_corrupt=0.5951 loss_corrupt=0.8767 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5103 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9134 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7881 out_g_norm=0.3632 loss_all=0.8128 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=14450 epoch=14450/14500 epoch_step=1/1 micro_steps=14450 elapsed=5.5s lr=2.000000e-03 loss=0.8609 loss_recon=0.8609 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5991 corrupt_frac=1.0000 acc_corrupt=0.5991 loss_corrupt=0.8609 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5143 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7902 out_g_norm=0.3422 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0289 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=14500 epoch=14500/14500 epoch_step=1/1 micro_steps=14500 elapsed=5.5s lr=2.000000e-03 loss=0.8408 loss_recon=0.8408 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6020 corrupt_frac=1.0000 acc_corrupt=0.6020 loss_corrupt=0.8408 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5212 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6750 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9148 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7939 out_g_norm=0.3328 acc_corrupt_t_0p8_1p0=0.9575 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8779 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=14501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=14550 epoch=14550/15000 epoch_step=1/1 micro_steps=14550 elapsed=6.2s lr=2.000000e-03 loss=0.8528 loss_recon=0.8528 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8528 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5186 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8056 out_g_norm=0.3359 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8559 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=14600 epoch=14600/15000 epoch_step=1/1 micro_steps=14600 elapsed=5.5s lr=2.000000e-03 loss=0.8550 loss_recon=0.8550 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5998 corrupt_frac=1.0000 acc_corrupt=0.5998 loss_corrupt=0.8550 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5153 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7994 out_g_norm=0.3481 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8949 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=14650 epoch=14650/15000 epoch_step=1/1 micro_steps=14650 elapsed=5.5s lr=2.000000e-03 loss=0.8695 loss_recon=0.8695 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5951 corrupt_frac=1.0000 acc_corrupt=0.5951 loss_corrupt=0.8695 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5139 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6731 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7897 out_g_norm=0.3427 acc_corrupt_t_0p8_1p0=0.9769 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9318 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=14700 epoch=14700/15000 epoch_step=1/1 micro_steps=14700 elapsed=5.5s lr=2.000000e-03 loss=0.8581 loss_recon=0.8581 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6000 corrupt_frac=1.0000 acc_corrupt=0.6000 loss_corrupt=0.8581 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5166 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7823 out_g_norm=0.3366 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8515 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=14750 epoch=14750/15000 epoch_step=1/1 micro_steps=14750 elapsed=5.5s lr=2.000000e-03 loss=0.8501 loss_recon=0.8501 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6012 corrupt_frac=1.0000 acc_corrupt=0.6012 loss_corrupt=0.8501 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5173 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7845 out_g_norm=0.3591 acc_corrupt_t_0p8_1p0=0.9751 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9036 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=14800 epoch=14800/15000 epoch_step=1/1 micro_steps=14800 elapsed=5.4s lr=2.000000e-03 loss=0.8546 loss_recon=0.8546 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6011 corrupt_frac=1.0000 acc_corrupt=0.6011 loss_corrupt=0.8546 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5155 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9064 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7831 out_g_norm=0.3935 loss_all=0.8392 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=14850 epoch=14850/15000 epoch_step=1/1 micro_steps=14850 elapsed=5.5s lr=2.000000e-03 loss=0.8610 loss_recon=0.8610 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5973 corrupt_frac=1.0000 acc_corrupt=0.5973 loss_corrupt=0.8610 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5182 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9054 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7762 out_g_norm=0.3597 loss_all=0.8065 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=14900 epoch=14900/15000 epoch_step=1/1 micro_steps=14900 elapsed=5.5s lr=2.000000e-03 loss=0.8499 loss_recon=0.8499 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6001 corrupt_frac=1.0000 acc_corrupt=0.6001 loss_corrupt=0.8499 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7707 out_g_norm=0.3123 loss_all=0.8005 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=14950 epoch=14950/15000 epoch_step=1/1 micro_steps=14950 elapsed=5.5s lr=2.000000e-03 loss=0.8682 loss_recon=0.8682 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5980 corrupt_frac=1.0000 acc_corrupt=0.5980 loss_corrupt=0.8682 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7713 out_g_norm=0.3507 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9893 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=15000 epoch=15000/15000 epoch_step=1/1 micro_steps=15000 elapsed=5.5s lr=2.000000e-03 loss=0.8634 loss_recon=0.8634 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5984 corrupt_frac=1.0000 acc_corrupt=0.5984 loss_corrupt=0.8634 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5153 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7619 out_g_norm=0.3531 acc_corrupt_t_0p8_1p0=0.9575 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9119 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=15001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=15050 epoch=15050/15500 epoch_step=1/1 micro_steps=15050 elapsed=6.2s lr=2.000000e-03 loss=0.8403 loss_recon=0.8403 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6005 corrupt_frac=1.0000 acc_corrupt=0.6005 loss_corrupt=0.8403 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5242 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7716 out_g_norm=0.3318 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8036 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=15100 epoch=15100/15500 epoch_step=1/1 micro_steps=15100 elapsed=5.4s lr=2.000000e-03 loss=0.8465 loss_recon=0.8465 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6020 corrupt_frac=1.0000 acc_corrupt=0.6020 loss_corrupt=0.8465 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5194 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7804 out_g_norm=0.3226 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8790 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=15150 epoch=15150/15500 epoch_step=1/1 micro_steps=15150 elapsed=5.4s lr=2.000000e-03 loss=0.8753 loss_recon=0.8753 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5943 corrupt_frac=1.0000 acc_corrupt=0.5943 loss_corrupt=0.8753 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5125 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7591 out_g_norm=0.3400 acc_corrupt_t_0p8_1p0=0.9798 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9084 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=15200 epoch=15200/15500 epoch_step=1/1 micro_steps=15200 elapsed=5.4s lr=2.000000e-03 loss=0.8509 loss_recon=0.8509 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6009 corrupt_frac=1.0000 acc_corrupt=0.6009 loss_corrupt=0.8509 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5183 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7564 out_g_norm=0.3626 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8712 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=15250 epoch=15250/15500 epoch_step=1/1 micro_steps=15250 elapsed=5.4s lr=2.000000e-03 loss=0.8462 loss_recon=0.8462 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6011 corrupt_frac=1.0000 acc_corrupt=0.6011 loss_corrupt=0.8462 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7658 out_g_norm=0.3710 acc_corrupt_t_0p8_1p0=0.9692 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9080 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=15300 epoch=15300/15500 epoch_step=1/1 micro_steps=15300 elapsed=5.4s lr=2.000000e-03 loss=0.8518 loss_recon=0.8518 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6015 corrupt_frac=1.0000 acc_corrupt=0.6015 loss_corrupt=0.8518 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5160 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9075 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7747 out_g_norm=0.3657 loss_all=0.7912 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=15350 epoch=15350/15500 epoch_step=1/1 micro_steps=15350 elapsed=5.5s lr=2.000000e-03 loss=0.8567 loss_recon=0.8567 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5980 corrupt_frac=1.0000 acc_corrupt=0.5980 loss_corrupt=0.8567 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5187 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7854 out_g_norm=0.3553 loss_all=0.7993 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=15400 epoch=15400/15500 epoch_step=1/1 micro_steps=15400 elapsed=5.5s lr=2.000000e-03 loss=0.8574 loss_recon=0.8574 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5985 corrupt_frac=1.0000 acc_corrupt=0.5985 loss_corrupt=0.8574 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5162 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7835 out_g_norm=0.3312 loss_all=0.8256 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=15450 epoch=15450/15500 epoch_step=1/1 micro_steps=15450 elapsed=5.5s lr=2.000000e-03 loss=0.8613 loss_recon=0.8613 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5991 corrupt_frac=1.0000 acc_corrupt=0.5991 loss_corrupt=0.8613 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5142 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7850 out_g_norm=0.3586 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9316 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=15500 epoch=15500/15500 epoch_step=1/1 micro_steps=15500 elapsed=5.5s lr=2.000000e-03 loss=0.8641 loss_recon=0.8641 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5981 corrupt_frac=1.0000 acc_corrupt=0.5981 loss_corrupt=0.8641 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5144 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9155 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7843 out_g_norm=0.3349 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8485 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=15501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=15550 epoch=15550/16000 epoch_step=1/1 micro_steps=15550 elapsed=6.3s lr=2.000000e-03 loss=0.8731 loss_recon=0.8731 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5945 corrupt_frac=1.0000 acc_corrupt=0.5945 loss_corrupt=0.8731 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5137 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7729 out_g_norm=0.3527 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8059 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=15600 epoch=15600/16000 epoch_step=1/1 micro_steps=15600 elapsed=5.4s lr=2.000000e-03 loss=0.8531 loss_recon=0.8531 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6013 corrupt_frac=1.0000 acc_corrupt=0.6013 loss_corrupt=0.8531 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5182 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7716 out_g_norm=0.3540 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8639 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=15650 epoch=15650/16000 epoch_step=1/1 micro_steps=15650 elapsed=5.5s lr=2.000000e-03 loss=0.8578 loss_recon=0.8578 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8578 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5181 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7716 out_g_norm=0.3360 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8891 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=15700 epoch=15700/16000 epoch_step=1/1 micro_steps=15700 elapsed=5.5s lr=2.000000e-03 loss=0.8586 loss_recon=0.8586 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5994 corrupt_frac=1.0000 acc_corrupt=0.5994 loss_corrupt=0.8586 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5159 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7705 out_g_norm=0.3236 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9504 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=15750 epoch=15750/16000 epoch_step=1/1 micro_steps=15750 elapsed=5.5s lr=2.000000e-03 loss=0.8569 loss_recon=0.8569 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6005 corrupt_frac=1.0000 acc_corrupt=0.6005 loss_corrupt=0.8569 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5164 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7736 out_g_norm=0.3634 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8896 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=15800 epoch=15800/16000 epoch_step=1/1 micro_steps=15800 elapsed=5.5s lr=2.000000e-03 loss=0.8497 loss_recon=0.8497 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6016 corrupt_frac=1.0000 acc_corrupt=0.6016 loss_corrupt=0.8497 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5163 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9073 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7715 out_g_norm=0.3479 loss_all=0.8314 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=15850 epoch=15850/16000 epoch_step=1/1 micro_steps=15850 elapsed=5.5s lr=2.000000e-03 loss=0.8531 loss_recon=0.8531 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5990 corrupt_frac=1.0000 acc_corrupt=0.5990 loss_corrupt=0.8531 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5204 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9062 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7849 out_g_norm=0.3207 loss_all=0.8724 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=15900 epoch=15900/16000 epoch_step=1/1 micro_steps=15900 elapsed=5.6s lr=2.000000e-03 loss=0.8526 loss_recon=0.8526 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5991 corrupt_frac=1.0000 acc_corrupt=0.5991 loss_corrupt=0.8526 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5170 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9126 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7915 out_g_norm=0.3410 loss_all=0.8089 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=15950 epoch=15950/16000 epoch_step=1/1 micro_steps=15950 elapsed=5.6s lr=2.000000e-03 loss=0.8557 loss_recon=0.8557 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6002 corrupt_frac=1.0000 acc_corrupt=0.6002 loss_corrupt=0.8557 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5164 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7863 out_g_norm=0.3886 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9883 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=16000 epoch=16000/16000 epoch_step=1/1 micro_steps=16000 elapsed=5.6s lr=2.000000e-03 loss=0.8426 loss_recon=0.8426 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6013 corrupt_frac=1.0000 acc_corrupt=0.6013 loss_corrupt=0.8426 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5202 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9152 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7893 out_g_norm=0.3490 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8240 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=16001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=16050 epoch=16050/16500 epoch_step=1/1 micro_steps=16050 elapsed=6.3s lr=2.000000e-03 loss=0.8636 loss_recon=0.8636 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5964 corrupt_frac=1.0000 acc_corrupt=0.5964 loss_corrupt=0.8636 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7983 out_g_norm=0.3529 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8012 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=16100 epoch=16100/16500 epoch_step=1/1 micro_steps=16100 elapsed=5.4s lr=2.000000e-03 loss=0.8562 loss_recon=0.8562 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6003 corrupt_frac=1.0000 acc_corrupt=0.6003 loss_corrupt=0.8562 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5161 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7974 out_g_norm=0.2953 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9351 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=16150 epoch=16150/16500 epoch_step=1/1 micro_steps=16150 elapsed=5.4s lr=2.000000e-03 loss=0.8586 loss_recon=0.8586 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5971 corrupt_frac=1.0000 acc_corrupt=0.5971 loss_corrupt=0.8586 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5165 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8150 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8014 out_g_norm=0.3142 acc_corrupt_t_0p8_1p0=0.9759 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8777 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=16200 epoch=16200/16500 epoch_step=1/1 micro_steps=16200 elapsed=5.4s lr=2.000000e-03 loss=0.8538 loss_recon=0.8538 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6008 corrupt_frac=1.0000 acc_corrupt=0.6008 loss_corrupt=0.8538 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5179 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7922 out_g_norm=0.3174 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9615 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=16250 epoch=16250/16500 epoch_step=1/1 micro_steps=16250 elapsed=5.4s lr=2.000000e-03 loss=0.8408 loss_recon=0.8408 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6022 corrupt_frac=1.0000 acc_corrupt=0.6022 loss_corrupt=0.8408 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5192 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7851 out_g_norm=0.3687 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8076 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=16300 epoch=16300/16500 epoch_step=1/1 micro_steps=16300 elapsed=5.4s lr=2.000000e-03 loss=0.8544 loss_recon=0.8544 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6013 corrupt_frac=1.0000 acc_corrupt=0.6013 loss_corrupt=0.8544 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5155 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8155 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9074 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7854 out_g_norm=0.3616 loss_all=0.8659 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=16350 epoch=16350/16500 epoch_step=1/1 micro_steps=16350 elapsed=5.4s lr=2.000000e-03 loss=0.8531 loss_recon=0.8531 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5986 corrupt_frac=1.0000 acc_corrupt=0.5986 loss_corrupt=0.8531 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5197 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9049 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7841 out_g_norm=0.3735 loss_all=0.7995 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=16400 epoch=16400/16500 epoch_step=1/1 micro_steps=16400 elapsed=5.4s lr=2.000000e-03 loss=0.8528 loss_recon=0.8528 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5988 corrupt_frac=1.0000 acc_corrupt=0.5988 loss_corrupt=0.8528 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5167 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8143 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9135 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7869 out_g_norm=0.3474 loss_all=0.7940 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=16450 epoch=16450/16500 epoch_step=1/1 micro_steps=16450 elapsed=5.4s lr=2.000000e-03 loss=0.8511 loss_recon=0.8511 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6010 corrupt_frac=1.0000 acc_corrupt=0.6010 loss_corrupt=0.8511 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5173 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7930 out_g_norm=0.3130 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9749 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=16500 epoch=16500/16500 epoch_step=1/1 micro_steps=16500 elapsed=5.4s lr=2.000000e-03 loss=0.8675 loss_recon=0.8675 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8675 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5127 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8015 out_g_norm=0.3316 acc_corrupt_t_0p8_1p0=0.9634 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9595 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=16501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=16550 epoch=16550/17000 epoch_step=1/1 micro_steps=16550 elapsed=6.3s lr=2.000000e-03 loss=0.8648 loss_recon=0.8648 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5964 corrupt_frac=1.0000 acc_corrupt=0.5964 loss_corrupt=0.8648 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7963 out_g_norm=0.3258 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8034 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=16600 epoch=16600/17000 epoch_step=1/1 micro_steps=16600 elapsed=5.5s lr=2.000000e-03 loss=0.8588 loss_recon=0.8588 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5995 corrupt_frac=1.0000 acc_corrupt=0.5995 loss_corrupt=0.8588 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5156 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8097 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7962 out_g_norm=0.3654 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8984 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=16650 epoch=16650/17000 epoch_step=1/1 micro_steps=16650 elapsed=5.5s lr=2.000000e-03 loss=0.8729 loss_recon=0.8729 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5950 corrupt_frac=1.0000 acc_corrupt=0.5950 loss_corrupt=0.8729 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5136 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7879 out_g_norm=0.3442 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0271 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=16700 epoch=16700/17000 epoch_step=1/1 micro_steps=16700 elapsed=5.5s lr=2.000000e-03 loss=0.8497 loss_recon=0.8497 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6010 corrupt_frac=1.0000 acc_corrupt=0.6010 loss_corrupt=0.8497 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5183 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7851 out_g_norm=0.3279 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9084 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=16750 epoch=16750/17000 epoch_step=1/1 micro_steps=16750 elapsed=5.6s lr=2.000000e-03 loss=0.8480 loss_recon=0.8480 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6020 corrupt_frac=1.0000 acc_corrupt=0.6020 loss_corrupt=0.8480 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5190 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7978 out_g_norm=0.3395 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8830 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=16800 epoch=16800/17000 epoch_step=1/1 micro_steps=16800 elapsed=5.7s lr=2.000000e-03 loss=0.8432 loss_recon=0.8432 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6032 corrupt_frac=1.0000 acc_corrupt=0.6032 loss_corrupt=0.8432 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5189 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7922 out_g_norm=0.3263 loss_all=0.7908 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=16850 epoch=16850/17000 epoch_step=1/1 micro_steps=16850 elapsed=5.6s lr=2.000000e-03 loss=0.8525 loss_recon=0.8525 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5984 corrupt_frac=1.0000 acc_corrupt=0.5984 loss_corrupt=0.8525 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5195 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9065 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7896 out_g_norm=0.3294 loss_all=0.8477 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=16900 epoch=16900/17000 epoch_step=1/1 micro_steps=16900 elapsed=5.5s lr=2.000000e-03 loss=0.8582 loss_recon=0.8582 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5979 corrupt_frac=1.0000 acc_corrupt=0.5979 loss_corrupt=0.8582 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5152 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9125 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7915 out_g_norm=0.3284 loss_all=0.9056 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=16950 epoch=16950/17000 epoch_step=1/1 micro_steps=16950 elapsed=5.5s lr=2.000000e-03 loss=0.8526 loss_recon=0.8526 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6012 corrupt_frac=1.0000 acc_corrupt=0.6012 loss_corrupt=0.8526 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5180 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7908 out_g_norm=0.3508 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9924 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=17000 epoch=17000/17000 epoch_step=1/1 micro_steps=17000 elapsed=5.5s lr=2.000000e-03 loss=0.8474 loss_recon=0.8474 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6015 corrupt_frac=1.0000 acc_corrupt=0.6015 loss_corrupt=0.8474 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5206 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9152 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7879 out_g_norm=0.3687 acc_corrupt_t_0p8_1p0=0.9609 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8590 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=17001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=17050 epoch=17050/17500 epoch_step=1/1 micro_steps=17050 elapsed=6.2s lr=2.000000e-03 loss=0.8438 loss_recon=0.8438 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5993 corrupt_frac=1.0000 acc_corrupt=0.5993 loss_corrupt=0.8438 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5218 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7889 out_g_norm=0.3235 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7611 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=17100 epoch=17100/17500 epoch_step=1/1 micro_steps=17100 elapsed=5.7s lr=2.000000e-03 loss=0.8493 loss_recon=0.8493 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6007 corrupt_frac=1.0000 acc_corrupt=0.6007 loss_corrupt=0.8493 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5167 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7837 out_g_norm=0.3446 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8869 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=17150 epoch=17150/17500 epoch_step=1/1 micro_steps=17150 elapsed=5.8s lr=2.000000e-03 loss=0.8563 loss_recon=0.8563 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5973 corrupt_frac=1.0000 acc_corrupt=0.5973 loss_corrupt=0.8563 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8146 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7781 out_g_norm=0.3103 acc_corrupt_t_0p8_1p0=0.9782 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8894 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=17200 epoch=17200/17500 epoch_step=1/1 micro_steps=17200 elapsed=5.6s lr=2.000000e-03 loss=0.8240 loss_recon=0.8240 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6056 corrupt_frac=1.0000 acc_corrupt=0.6056 loss_corrupt=0.8240 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5262 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7734 out_g_norm=0.3357 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8994 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=17250 epoch=17250/17500 epoch_step=1/1 micro_steps=17250 elapsed=5.5s lr=2.000000e-03 loss=0.8372 loss_recon=0.8372 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6032 corrupt_frac=1.0000 acc_corrupt=0.6032 loss_corrupt=0.8372 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5202 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7691 out_g_norm=0.3379 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8859 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=17300 epoch=17300/17500 epoch_step=1/1 micro_steps=17300 elapsed=5.5s lr=2.000000e-03 loss=0.8466 loss_recon=0.8466 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6023 corrupt_frac=1.0000 acc_corrupt=0.6023 loss_corrupt=0.8466 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8153 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9076 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7521 out_g_norm=0.3127 loss_all=0.7656 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=17350 epoch=17350/17500 epoch_step=1/1 micro_steps=17350 elapsed=5.5s lr=2.000000e-03 loss=0.8556 loss_recon=0.8556 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5986 corrupt_frac=1.0000 acc_corrupt=0.5986 loss_corrupt=0.8556 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5197 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9056 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7452 out_g_norm=0.3338 loss_all=0.7650 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=17400 epoch=17400/17500 epoch_step=1/1 micro_steps=17400 elapsed=5.7s lr=2.000000e-03 loss=0.8459 loss_recon=0.8459 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6001 corrupt_frac=1.0000 acc_corrupt=0.6001 loss_corrupt=0.8459 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5183 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7456 out_g_norm=0.3175 loss_all=0.8521 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=17450 epoch=17450/17500 epoch_step=1/1 micro_steps=17450 elapsed=5.7s lr=2.000000e-03 loss=0.8479 loss_recon=0.8479 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6021 corrupt_frac=1.0000 acc_corrupt=0.6021 loss_corrupt=0.8479 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5191 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7460 out_g_norm=0.3543 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9770 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=17500 epoch=17500/17500 epoch_step=1/1 micro_steps=17500 elapsed=5.8s lr=2.000000e-03 loss=0.8444 loss_recon=0.8444 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6012 corrupt_frac=1.0000 acc_corrupt=0.6012 loss_corrupt=0.8444 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5198 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6751 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9158 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7445 out_g_norm=0.3320 acc_corrupt_t_0p8_1p0=0.9634 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8854 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=17501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=17550 epoch=17550/18000 epoch_step=1/1 micro_steps=17550 elapsed=6.2s lr=2.000000e-03 loss=0.8729 loss_recon=0.8729 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5948 corrupt_frac=1.0000 acc_corrupt=0.5948 loss_corrupt=0.8729 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5143 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7334 out_g_norm=0.3497 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7800 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=17600 epoch=17600/18000 epoch_step=1/1 micro_steps=17600 elapsed=5.4s lr=2.000000e-03 loss=0.8395 loss_recon=0.8395 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6028 corrupt_frac=1.0000 acc_corrupt=0.6028 loss_corrupt=0.8395 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5206 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7288 out_g_norm=0.3232 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8756 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=17650 epoch=17650/18000 epoch_step=1/1 micro_steps=17650 elapsed=5.4s lr=2.000000e-03 loss=0.8500 loss_recon=0.8500 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5983 corrupt_frac=1.0000 acc_corrupt=0.5983 loss_corrupt=0.8500 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8143 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7341 out_g_norm=0.3509 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8629 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=17700 epoch=17700/18000 epoch_step=1/1 micro_steps=17700 elapsed=5.4s lr=2.000000e-03 loss=0.8356 loss_recon=0.8356 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6035 corrupt_frac=1.0000 acc_corrupt=0.6035 loss_corrupt=0.8356 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5221 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7414 out_g_norm=0.2983 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8331 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=17750 epoch=17750/18000 epoch_step=1/1 micro_steps=17750 elapsed=5.4s lr=2.000000e-03 loss=0.8441 loss_recon=0.8441 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6022 corrupt_frac=1.0000 acc_corrupt=0.6022 loss_corrupt=0.8441 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5186 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7404 out_g_norm=0.3285 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8415 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=17800 epoch=17800/18000 epoch_step=1/1 micro_steps=17800 elapsed=5.4s lr=2.000000e-03 loss=0.8309 loss_recon=0.8309 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6052 corrupt_frac=1.0000 acc_corrupt=0.6052 loss_corrupt=0.8309 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5220 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8156 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9076 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7305 out_g_norm=0.3224 loss_all=0.8359 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=17850 epoch=17850/18000 epoch_step=1/1 micro_steps=17850 elapsed=5.4s lr=2.000000e-03 loss=0.8551 loss_recon=0.8551 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5980 corrupt_frac=1.0000 acc_corrupt=0.5980 loss_corrupt=0.8551 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5185 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9063 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7270 out_g_norm=0.3136 loss_all=0.7718 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=17900 epoch=17900/18000 epoch_step=1/1 micro_steps=17900 elapsed=5.4s lr=2.000000e-03 loss=0.8610 loss_recon=0.8610 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5976 corrupt_frac=1.0000 acc_corrupt=0.5976 loss_corrupt=0.8610 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5148 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7152 out_g_norm=0.3277 loss_all=0.8178 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=17950 epoch=17950/18000 epoch_step=1/1 micro_steps=17950 elapsed=5.4s lr=2.000000e-03 loss=0.8441 loss_recon=0.8441 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6025 corrupt_frac=1.0000 acc_corrupt=0.6025 loss_corrupt=0.8441 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5198 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7178 out_g_norm=0.3275 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0745 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=18000 epoch=18000/18000 epoch_step=1/1 micro_steps=18000 elapsed=5.4s lr=2.000000e-03 loss=0.8572 loss_recon=0.8572 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5984 corrupt_frac=1.0000 acc_corrupt=0.5984 loss_corrupt=0.8572 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5150 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9157 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7134 out_g_norm=0.3251 acc_corrupt_t_0p8_1p0=0.9619 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9192 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=18001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=18050 epoch=18050/18500 epoch_step=1/1 micro_steps=18050 elapsed=6.7s lr=2.000000e-03 loss=0.8607 loss_recon=0.8607 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5973 corrupt_frac=1.0000 acc_corrupt=0.5973 loss_corrupt=0.8607 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5186 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6722 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7076 out_g_norm=0.3778 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7920 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=18100 epoch=18100/18500 epoch_step=1/1 micro_steps=18100 elapsed=5.4s lr=2.000000e-03 loss=0.8491 loss_recon=0.8491 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6012 corrupt_frac=1.0000 acc_corrupt=0.6012 loss_corrupt=0.8491 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5177 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7054 out_g_norm=0.3507 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8335 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=18150 epoch=18150/18500 epoch_step=1/1 micro_steps=18150 elapsed=5.4s lr=2.000000e-03 loss=0.8541 loss_recon=0.8541 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8541 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5169 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7065 out_g_norm=0.3405 acc_corrupt_t_0p8_1p0=0.9782 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8912 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=18200 epoch=18200/18500 epoch_step=1/1 micro_steps=18200 elapsed=5.4s lr=2.000000e-03 loss=0.8437 loss_recon=0.8437 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6019 corrupt_frac=1.0000 acc_corrupt=0.6019 loss_corrupt=0.8437 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5196 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7067 out_g_norm=0.3038 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9022 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=18250 epoch=18250/18500 epoch_step=1/1 micro_steps=18250 elapsed=5.4s lr=2.000000e-03 loss=0.8268 loss_recon=0.8268 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6054 corrupt_frac=1.0000 acc_corrupt=0.6054 loss_corrupt=0.8268 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5245 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7036 out_g_norm=0.3181 acc_corrupt_t_0p8_1p0=0.9731 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8668 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=18300 epoch=18300/18500 epoch_step=1/1 micro_steps=18300 elapsed=5.4s lr=2.000000e-03 loss=0.8394 loss_recon=0.8394 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6035 corrupt_frac=1.0000 acc_corrupt=0.6035 loss_corrupt=0.8394 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8156 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9060 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7105 out_g_norm=0.3649 loss_all=0.8226 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=18350 epoch=18350/18500 epoch_step=1/1 micro_steps=18350 elapsed=5.4s lr=2.000000e-03 loss=0.8467 loss_recon=0.8467 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5997 corrupt_frac=1.0000 acc_corrupt=0.5997 loss_corrupt=0.8467 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5216 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9050 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6998 out_g_norm=0.3540 loss_all=0.8030 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=18400 epoch=18400/18500 epoch_step=1/1 micro_steps=18400 elapsed=5.4s lr=2.000000e-03 loss=0.8538 loss_recon=0.8538 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5984 corrupt_frac=1.0000 acc_corrupt=0.5984 loss_corrupt=0.8538 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5162 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6922 out_g_norm=0.3466 loss_all=0.8150 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=18450 epoch=18450/18500 epoch_step=1/1 micro_steps=18450 elapsed=5.4s lr=2.000000e-03 loss=0.8589 loss_recon=0.8589 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5998 corrupt_frac=1.0000 acc_corrupt=0.5998 loss_corrupt=0.8589 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5155 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6734 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6945 out_g_norm=0.3432 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0148 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=18500 epoch=18500/18500 epoch_step=1/1 micro_steps=18500 elapsed=5.4s lr=2.000000e-03 loss=0.8424 loss_recon=0.8424 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6017 corrupt_frac=1.0000 acc_corrupt=0.6017 loss_corrupt=0.8424 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5210 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6982 out_g_norm=0.3454 acc_corrupt_t_0p8_1p0=0.9634 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9173 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=18501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=18550 epoch=18550/19000 epoch_step=1/1 micro_steps=18550 elapsed=6.3s lr=2.000000e-03 loss=0.8565 loss_recon=0.8565 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5977 corrupt_frac=1.0000 acc_corrupt=0.5977 loss_corrupt=0.8565 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6963 out_g_norm=0.3331 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7696 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=18600 epoch=18600/19000 epoch_step=1/1 micro_steps=18600 elapsed=5.5s lr=2.000000e-03 loss=0.8445 loss_recon=0.8445 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6019 corrupt_frac=1.0000 acc_corrupt=0.6019 loss_corrupt=0.8445 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7012 out_g_norm=0.3238 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8924 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=18650 epoch=18650/19000 epoch_step=1/1 micro_steps=18650 elapsed=5.5s lr=2.000000e-03 loss=0.8539 loss_recon=0.8539 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5983 corrupt_frac=1.0000 acc_corrupt=0.5983 loss_corrupt=0.8539 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6950 out_g_norm=0.3205 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9404 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=18700 epoch=18700/19000 epoch_step=1/1 micro_steps=18700 elapsed=5.6s lr=2.000000e-03 loss=0.8534 loss_recon=0.8534 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6009 corrupt_frac=1.0000 acc_corrupt=0.6009 loss_corrupt=0.8534 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5182 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6873 out_g_norm=0.3403 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9491 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=18750 epoch=18750/19000 epoch_step=1/1 micro_steps=18750 elapsed=5.5s lr=2.000000e-03 loss=0.8420 loss_recon=0.8420 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6024 corrupt_frac=1.0000 acc_corrupt=0.6024 loss_corrupt=0.8420 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5195 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6759 out_g_norm=0.3460 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9013 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=18800 epoch=18800/19000 epoch_step=1/1 micro_steps=18800 elapsed=5.5s lr=2.000000e-03 loss=0.8463 loss_recon=0.8463 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6023 corrupt_frac=1.0000 acc_corrupt=0.6023 loss_corrupt=0.8463 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8156 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6705 out_g_norm=0.3471 loss_all=0.8137 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=18850 epoch=18850/19000 epoch_step=1/1 micro_steps=18850 elapsed=5.5s lr=2.000000e-03 loss=0.8584 loss_recon=0.8584 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5978 corrupt_frac=1.0000 acc_corrupt=0.5978 loss_corrupt=0.8584 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5185 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9052 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6666 out_g_norm=0.3131 loss_all=0.8086 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=18900 epoch=18900/19000 epoch_step=1/1 micro_steps=18900 elapsed=5.5s lr=2.000000e-03 loss=0.8339 loss_recon=0.8339 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6025 corrupt_frac=1.0000 acc_corrupt=0.6025 loss_corrupt=0.8339 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5226 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8141 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6676 out_g_norm=0.3123 loss_all=0.7870 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=18950 epoch=18950/19000 epoch_step=1/1 micro_steps=18950 elapsed=5.5s lr=2.000000e-03 loss=0.8389 loss_recon=0.8389 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6029 corrupt_frac=1.0000 acc_corrupt=0.6029 loss_corrupt=0.8389 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5206 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6804 out_g_norm=0.3097 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0411 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=19000 epoch=19000/19000 epoch_step=1/1 micro_steps=19000 elapsed=5.5s lr=2.000000e-03 loss=0.8426 loss_recon=0.8426 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6010 corrupt_frac=1.0000 acc_corrupt=0.6010 loss_corrupt=0.8426 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5194 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9159 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6768 out_g_norm=0.3114 acc_corrupt_t_0p8_1p0=0.9619 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8574 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=19001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=19050 epoch=19050/19500 epoch_step=1/1 micro_steps=19050 elapsed=6.2s lr=2.000000e-03 loss=0.8381 loss_recon=0.8381 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6007 corrupt_frac=1.0000 acc_corrupt=0.6007 loss_corrupt=0.8381 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5235 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6738 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8143 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6770 out_g_norm=0.3205 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8302 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=19100 epoch=19100/19500 epoch_step=1/1 micro_steps=19100 elapsed=5.4s lr=2.000000e-03 loss=0.8401 loss_recon=0.8401 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6027 corrupt_frac=1.0000 acc_corrupt=0.6027 loss_corrupt=0.8401 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5201 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6750 out_g_norm=0.3141 acc_corrupt_t_0p6_0p8=0.9127 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8631 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=19150 epoch=19150/19500 epoch_step=1/1 micro_steps=19150 elapsed=5.4s lr=2.000000e-03 loss=0.8406 loss_recon=0.8406 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6008 corrupt_frac=1.0000 acc_corrupt=0.6008 loss_corrupt=0.8406 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5224 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8155 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6704 out_g_norm=0.2930 acc_corrupt_t_0p8_1p0=0.9762 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9064 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=19200 epoch=19200/19500 epoch_step=1/1 micro_steps=19200 elapsed=5.5s lr=2.000000e-03 loss=0.8325 loss_recon=0.8325 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6035 corrupt_frac=1.0000 acc_corrupt=0.6035 loss_corrupt=0.8325 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5223 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6763 out_g_norm=0.3100 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9177 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=19250 epoch=19250/19500 epoch_step=1/1 micro_steps=19250 elapsed=5.4s lr=2.000000e-03 loss=0.8236 loss_recon=0.8236 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6049 corrupt_frac=1.0000 acc_corrupt=0.6049 loss_corrupt=0.8236 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5236 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6661 out_g_norm=0.3264 acc_corrupt_t_0p8_1p0=0.9731 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8702 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=19300 epoch=19300/19500 epoch_step=1/1 micro_steps=19300 elapsed=5.4s lr=2.000000e-03 loss=0.8322 loss_recon=0.8322 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6047 corrupt_frac=1.0000 acc_corrupt=0.6047 loss_corrupt=0.8322 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5208 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8163 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9067 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6657 out_g_norm=0.3014 loss_all=0.8035 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=19350 epoch=19350/19500 epoch_step=1/1 micro_steps=19350 elapsed=5.5s lr=2.000000e-03 loss=0.8384 loss_recon=0.8384 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6016 corrupt_frac=1.0000 acc_corrupt=0.6016 loss_corrupt=0.8384 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5244 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9060 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6550 out_g_norm=0.3104 loss_all=0.8047 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=19400 epoch=19400/19500 epoch_step=1/1 micro_steps=19400 elapsed=5.5s lr=2.000000e-03 loss=0.8421 loss_recon=0.8421 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6007 corrupt_frac=1.0000 acc_corrupt=0.6007 loss_corrupt=0.8421 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5194 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6581 out_g_norm=0.3366 loss_all=0.8411 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=19450 epoch=19450/19500 epoch_step=1/1 micro_steps=19450 elapsed=5.5s lr=2.000000e-03 loss=0.8414 loss_recon=0.8414 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6030 corrupt_frac=1.0000 acc_corrupt=0.6030 loss_corrupt=0.8414 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5204 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6494 out_g_norm=0.3147 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9927 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=19500 epoch=19500/19500 epoch_step=1/1 micro_steps=19500 elapsed=5.5s lr=2.000000e-03 loss=0.8436 loss_recon=0.8436 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6015 corrupt_frac=1.0000 acc_corrupt=0.6015 loss_corrupt=0.8436 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5203 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9158 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6475 out_g_norm=0.3358 acc_corrupt_t_0p8_1p0=0.9624 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8557 init_gold_top10=0.1993 init_gold_top100=0.3202 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=19501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 20000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=19550 epoch=19550/20000 epoch_step=1/1 micro_steps=19550 elapsed=6.3s lr=2.000000e-03 loss=0.8476 loss_recon=0.8476 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5988 corrupt_frac=1.0000 acc_corrupt=0.5988 loss_corrupt=0.8476 wrong_frac=0.7943 init_acc_corrupt=0.1118 acc_corrupt_t_0p0_0p2=0.5207 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6449 out_g_norm=0.3107 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8840 init_gold_top10=0.2072 init_gold_top100=0.3278 +step=19600 epoch=19600/20000 epoch_step=1/1 micro_steps=19600 elapsed=5.5s lr=2.000000e-03 loss=0.8435 loss_recon=0.8435 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6024 corrupt_frac=1.0000 acc_corrupt=0.6024 loss_corrupt=0.8435 wrong_frac=0.7895 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.5198 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6426 out_g_norm=0.3103 acc_corrupt_t_0p6_0p8=0.9115 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9629 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8438 init_gold_top10=0.1787 init_gold_top100=0.3107 +step=19650 epoch=19650/20000 epoch_step=1/1 micro_steps=19650 elapsed=5.5s lr=2.000000e-03 loss=0.8596 loss_recon=0.8596 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5972 corrupt_frac=1.0000 acc_corrupt=0.5972 loss_corrupt=0.8596 wrong_frac=0.7941 init_acc_corrupt=0.1127 acc_corrupt_t_0p0_0p2=0.5169 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8153 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6437 out_g_norm=0.3231 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8858 init_gold_top10=0.1707 init_gold_top100=0.3132 +step=19700 epoch=19700/20000 epoch_step=1/1 micro_steps=19700 elapsed=5.5s lr=2.000000e-03 loss=0.8384 loss_recon=0.8384 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6030 corrupt_frac=1.0000 acc_corrupt=0.6030 loss_corrupt=0.8384 wrong_frac=0.7901 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5216 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.6751 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6374 out_g_norm=0.3425 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8684 init_gold_top10=0.1794 init_gold_top100=0.3123 +step=19750 epoch=19750/20000 epoch_step=1/1 micro_steps=19750 elapsed=5.5s lr=2.000000e-03 loss=0.8395 loss_recon=0.8395 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6024 corrupt_frac=1.0000 acc_corrupt=0.6024 loss_corrupt=0.8395 wrong_frac=0.7889 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5187 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.6750 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6390 out_g_norm=0.3079 acc_corrupt_t_0p8_1p0=0.9741 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9374 init_gold_top10=0.1933 init_gold_top100=0.3218 +step=19800 epoch=19800/20000 epoch_step=1/1 micro_steps=19800 elapsed=5.5s lr=2.000000e-03 loss=0.8357 loss_recon=0.8357 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6049 corrupt_frac=1.0000 acc_corrupt=0.6049 loss_corrupt=0.8357 wrong_frac=0.7877 init_acc_corrupt=0.1204 acc_corrupt_t_0p0_0p2=0.5220 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8157 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9060 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6310 out_g_norm=0.3361 loss_all=0.7675 init_gold_top10=0.2009 init_gold_top100=0.3269 +step=19850 epoch=19850/20000 epoch_step=1/1 micro_steps=19850 elapsed=5.5s lr=2.000000e-03 loss=0.8500 loss_recon=0.8500 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5992 corrupt_frac=1.0000 acc_corrupt=0.5992 loss_corrupt=0.8500 wrong_frac=0.7936 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.5204 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9047 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6267 out_g_norm=0.3088 loss_all=0.7812 init_gold_top10=0.2107 init_gold_top100=0.3333 +step=19900 epoch=19900/20000 epoch_step=1/1 micro_steps=19900 elapsed=5.5s lr=2.000000e-03 loss=0.8364 loss_recon=0.8364 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6015 corrupt_frac=1.0000 acc_corrupt=0.6015 loss_corrupt=0.8364 wrong_frac=0.7919 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.5207 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6750 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6219 out_g_norm=0.3110 loss_all=0.8968 init_gold_top10=0.1965 init_gold_top100=0.3234 +step=19950 epoch=19950/20000 epoch_step=1/1 micro_steps=19950 elapsed=5.5s lr=2.000000e-03 loss=0.8454 loss_recon=0.8454 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6013 corrupt_frac=1.0000 acc_corrupt=0.6013 loss_corrupt=0.8454 wrong_frac=0.7897 init_acc_corrupt=0.1177 acc_corrupt_t_0p0_0p2=0.5178 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6159 out_g_norm=0.3556 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8989 init_gold_top10=0.1803 init_gold_top100=0.3116 +step=20000 epoch=20000/20000 epoch_step=1/1 micro_steps=20000 elapsed=5.7s lr=2.000000e-03 loss=0.8366 loss_recon=0.8366 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6021 corrupt_frac=1.0000 acc_corrupt=0.6021 loss_corrupt=0.8366 wrong_frac=0.7907 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.5214 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9151 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6111 out_g_norm=0.3456 acc_corrupt_t_0p8_1p0=0.9614 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9292 init_gold_top10=0.1993 init_gold_top100=0.3202 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot.log new file mode 100644 index 0000000000000000000000000000000000000000..6bf9697907ee55afd7fedefcc4fce64532bebadc --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot.log @@ -0,0 +1,7919 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=50 epoch=50/500 epoch_step=1/1 micro_steps=50 elapsed=7.4s lr=2.000000e-03 loss=7.8266 loss_recon=7.8266 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0960 corrupt_frac=0.9977 acc_corrupt=0.0948 loss_corrupt=7.8266 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.0473 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.1290 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.2582 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.3775 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=0.4964 out_g_norm=0.7995 acc_corrupt_t_0p8_1p0=0.4919 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.7507 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=6.5s lr=2.000000e-03 loss=7.6142 loss_recon=7.6142 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0982 corrupt_frac=0.9977 acc_corrupt=0.0973 loss_corrupt=7.6142 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.0531 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.1281 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.2404 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.3329 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=1.5269 out_g_norm=1.3597 acc_corrupt_t_0p8_1p0=0.4933 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.4651 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=150 epoch=150/500 epoch_step=1/1 micro_steps=150 elapsed=6.5s lr=2.000000e-03 loss=7.2800 loss_recon=7.2800 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1017 corrupt_frac=0.9977 acc_corrupt=0.1008 loss_corrupt=7.2800 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.0545 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.1354 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.2501 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.3346 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=2.4394 out_g_norm=1.7220 loss_all=7.0786 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=6.5s lr=2.000000e-03 loss=6.8916 loss_recon=6.8916 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1074 corrupt_frac=0.9977 acc_corrupt=0.1064 loss_corrupt=6.8916 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.0566 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.1452 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.2638 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.3575 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=3.3395 out_g_norm=1.8223 acc_corrupt_t_0p8_1p0=0.5005 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.7053 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=250 epoch=250/500 epoch_step=1/1 micro_steps=250 elapsed=6.5s lr=2.000000e-03 loss=6.5840 loss_recon=6.5840 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1125 corrupt_frac=0.9977 acc_corrupt=0.1116 loss_corrupt=6.5840 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.0585 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.1517 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.2755 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=4.1207 out_g_norm=1.5656 acc_corrupt_t_0p6_0p8=0.3809 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.4235 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.4317 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=6.5s lr=2.000000e-03 loss=6.3173 loss_recon=6.3173 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1177 corrupt_frac=0.9977 acc_corrupt=0.1167 loss_corrupt=6.3173 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.0607 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.1605 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.2892 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.4016 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=4.6623 out_g_norm=1.0770 acc_corrupt_t_0p8_1p0=0.5900 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.0963 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=350 epoch=350/500 epoch_step=1/1 micro_steps=350 elapsed=6.5s lr=2.000000e-03 loss=6.0970 loss_recon=6.0970 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1217 corrupt_frac=0.9977 acc_corrupt=0.1207 loss_corrupt=6.0970 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.0627 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.1666 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.3085 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.4271 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=5.2271 out_g_norm=0.6212 loss_all=6.0291 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=6.5s lr=2.000000e-03 loss=5.8590 loss_recon=5.8590 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1305 corrupt_frac=0.9977 acc_corrupt=0.1295 loss_corrupt=5.8590 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.0653 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.1761 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.3255 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.4547 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=5.8811 out_g_norm=0.4149 acc_corrupt_t_0p8_1p0=0.5565 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.7609 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=450 epoch=450/500 epoch_step=1/1 micro_steps=450 elapsed=6.5s lr=2.000000e-03 loss=5.6270 loss_recon=5.6270 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1360 corrupt_frac=0.9977 acc_corrupt=0.1349 loss_corrupt=5.6270 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.0669 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.1850 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.3443 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=6.5298 out_g_norm=0.3746 acc_corrupt_t_0p6_0p8=0.4791 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.6572 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.5317 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=6.5s lr=2.000000e-03 loss=5.3520 loss_recon=5.3520 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1409 corrupt_frac=0.9977 acc_corrupt=0.1397 loss_corrupt=5.3520 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.0693 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.1952 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.3607 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=7.1695 out_g_norm=0.3838 acc_corrupt_t_0p6_0p8=0.5113 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.6069 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.0999 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=550 epoch=550/1000 epoch_step=1/1 micro_steps=550 elapsed=7.5s lr=2.000000e-03 loss=5.0260 loss_recon=5.0260 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1489 corrupt_frac=0.9977 acc_corrupt=0.1477 loss_corrupt=5.0260 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.0721 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.2087 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.3846 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.5332 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=7.8267 out_g_norm=0.4031 acc_corrupt_t_0p8_1p0=0.6802 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.8337 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=6.5s lr=2.000000e-03 loss=4.6636 loss_recon=4.6636 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1599 corrupt_frac=0.9977 acc_corrupt=0.1586 loss_corrupt=4.6636 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.0746 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.2218 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.4116 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.6026 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=8.4876 out_g_norm=0.4193 acc_corrupt_t_0p8_1p0=0.7732 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.4967 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=650 epoch=650/1000 epoch_step=1/1 micro_steps=650 elapsed=6.5s lr=2.000000e-03 loss=4.3539 loss_recon=4.3539 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1719 corrupt_frac=0.9977 acc_corrupt=0.1705 loss_corrupt=4.3539 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.0772 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.2397 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.4711 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.6642 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=9.0984 out_g_norm=0.4668 loss_all=4.2639 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=6.5s lr=2.000000e-03 loss=4.0728 loss_recon=4.0728 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1851 corrupt_frac=0.9977 acc_corrupt=0.1835 loss_corrupt=4.0728 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.0809 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.2645 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.5054 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.6747 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=9.6410 out_g_norm=0.4676 acc_corrupt_t_0p8_1p0=0.8211 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.8755 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=750 epoch=750/1000 epoch_step=1/1 micro_steps=750 elapsed=6.5s lr=2.000000e-03 loss=3.8368 loss_recon=3.8368 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1942 corrupt_frac=0.9977 acc_corrupt=0.1926 loss_corrupt=3.8368 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.0857 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.2775 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.5098 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=10.0859 out_g_norm=0.5826 acc_corrupt_t_0p6_0p8=0.6816 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.8500 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.7490 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=6.5s lr=2.000000e-03 loss=3.6367 loss_recon=3.6367 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2012 corrupt_frac=0.9977 acc_corrupt=0.1996 loss_corrupt=3.6367 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.0901 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.2911 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.5165 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.6834 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=10.4261 out_g_norm=0.7353 acc_corrupt_t_0p8_1p0=0.8620 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.3938 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=850 epoch=850/1000 epoch_step=1/1 micro_steps=850 elapsed=6.5s lr=2.000000e-03 loss=3.4778 loss_recon=3.4778 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2078 corrupt_frac=0.9977 acc_corrupt=0.2062 loss_corrupt=3.4778 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.0949 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.3034 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.5294 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.6959 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=10.6782 out_g_norm=0.8210 loss_all=3.4618 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=6.5s lr=2.000000e-03 loss=3.2542 loss_recon=3.2542 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2260 corrupt_frac=0.9977 acc_corrupt=0.2244 loss_corrupt=3.2542 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.1045 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.3238 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.5489 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.7101 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=10.8870 out_g_norm=0.8241 acc_corrupt_t_0p8_1p0=0.8474 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.1805 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=950 epoch=950/1000 epoch_step=1/1 micro_steps=950 elapsed=6.5s lr=2.000000e-03 loss=3.1024 loss_recon=3.1024 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2391 corrupt_frac=0.9977 acc_corrupt=0.2375 loss_corrupt=3.1024 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.1117 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.3455 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.5712 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.0680 out_g_norm=1.1751 acc_corrupt_t_0p6_0p8=0.7259 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.8639 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.0406 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=6.5s lr=2.000000e-03 loss=2.9325 loss_recon=2.9325 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2573 corrupt_frac=0.9977 acc_corrupt=0.2557 loss_corrupt=2.9325 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.1282 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.3745 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.5911 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.2066 out_g_norm=1.1671 acc_corrupt_t_0p6_0p8=0.7460 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.8510 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.7052 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=1050 epoch=1050/1500 epoch_step=1/1 micro_steps=1050 elapsed=7.6s lr=2.000000e-03 loss=2.7025 loss_recon=2.7025 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2888 corrupt_frac=0.9977 acc_corrupt=0.2873 loss_corrupt=2.7025 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.1554 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.4166 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.6220 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.7579 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3192 out_g_norm=1.2280 acc_corrupt_t_0p8_1p0=0.8988 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.5350 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=1100 epoch=1100/1500 epoch_step=1/1 micro_steps=1100 elapsed=6.6s lr=2.000000e-03 loss=2.3982 loss_recon=2.3982 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3354 corrupt_frac=0.9977 acc_corrupt=0.3340 loss_corrupt=2.3982 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.1954 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.4663 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.6520 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.7728 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4194 out_g_norm=1.3068 acc_corrupt_t_0p8_1p0=0.8995 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.3416 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=1150 epoch=1150/1500 epoch_step=1/1 micro_steps=1150 elapsed=6.6s lr=2.000000e-03 loss=2.1465 loss_recon=2.1465 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3781 corrupt_frac=0.9977 acc_corrupt=0.3768 loss_corrupt=2.1465 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.2414 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.5136 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.6783 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.7927 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5136 out_g_norm=1.5271 loss_all=2.0734 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=1200 epoch=1200/1500 epoch_step=1/1 micro_steps=1200 elapsed=6.6s lr=2.000000e-03 loss=1.8921 loss_recon=1.8921 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4240 corrupt_frac=0.9977 acc_corrupt=0.4228 loss_corrupt=1.8921 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.2920 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.5590 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7083 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8076 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5990 out_g_norm=1.5758 acc_corrupt_t_0p8_1p0=0.8954 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.7554 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=1250 epoch=1250/1500 epoch_step=1/1 micro_steps=1250 elapsed=6.6s lr=2.000000e-03 loss=1.7146 loss_recon=1.7146 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4535 corrupt_frac=0.9977 acc_corrupt=0.4524 loss_corrupt=1.7146 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.3236 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.5862 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7214 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6726 out_g_norm=1.6283 acc_corrupt_t_0p6_0p8=0.8200 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9010 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.6437 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=1300 epoch=1300/1500 epoch_step=1/1 micro_steps=1300 elapsed=6.6s lr=2.000000e-03 loss=1.5909 loss_recon=1.5909 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4723 corrupt_frac=0.9977 acc_corrupt=0.4713 loss_corrupt=1.5909 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.3458 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6029 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7292 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8261 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.7246 out_g_norm=1.7325 acc_corrupt_t_0p8_1p0=0.9227 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4817 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=1350 epoch=1350/1500 epoch_step=1/1 micro_steps=1350 elapsed=6.6s lr=2.000000e-03 loss=1.5412 loss_recon=1.5412 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4771 corrupt_frac=0.9977 acc_corrupt=0.4760 loss_corrupt=1.5412 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.3504 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6107 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7355 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8287 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.7442 out_g_norm=1.7222 loss_all=1.5570 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=1400 epoch=1400/1500 epoch_step=1/1 micro_steps=1400 elapsed=6.6s lr=2.000000e-03 loss=1.4398 loss_recon=1.4398 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4958 corrupt_frac=0.9977 acc_corrupt=0.4948 loss_corrupt=1.4398 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.3718 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6198 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7398 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8355 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.7447 out_g_norm=1.6109 acc_corrupt_t_0p8_1p0=0.9002 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4586 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=1450 epoch=1450/1500 epoch_step=1/1 micro_steps=1450 elapsed=6.6s lr=2.000000e-03 loss=1.4106 loss_recon=1.4106 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4997 corrupt_frac=0.9977 acc_corrupt=0.4987 loss_corrupt=1.4106 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.3763 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6243 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7441 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.7419 out_g_norm=1.5271 acc_corrupt_t_0p6_0p8=0.8360 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9187 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4548 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=1500 epoch=1500/1500 epoch_step=1/1 micro_steps=1500 elapsed=6.6s lr=2.000000e-03 loss=1.3956 loss_recon=1.3956 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5001 corrupt_frac=0.9977 acc_corrupt=0.4991 loss_corrupt=1.3956 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.3816 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6253 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7435 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.7291 out_g_norm=1.6351 acc_corrupt_t_0p6_0p8=0.8421 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9029 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2871 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=1501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=1550 epoch=1550/2000 epoch_step=1/1 micro_steps=1550 elapsed=7.4s lr=2.000000e-03 loss=1.3827 loss_recon=1.3827 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5030 corrupt_frac=0.9977 acc_corrupt=0.5020 loss_corrupt=1.3827 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.3867 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6282 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.7446 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8380 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.7124 out_g_norm=1.4729 acc_corrupt_t_0p8_1p0=0.9227 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2517 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=6.5s lr=2.000000e-03 loss=1.3194 loss_recon=1.3194 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5142 corrupt_frac=0.9977 acc_corrupt=0.5132 loss_corrupt=1.3194 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.3985 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6301 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.7470 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8362 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.7019 out_g_norm=1.2865 acc_corrupt_t_0p8_1p0=0.9308 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3881 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=1650 epoch=1650/2000 epoch_step=1/1 micro_steps=1650 elapsed=6.8s lr=2.000000e-03 loss=1.3025 loss_recon=1.3025 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5148 corrupt_frac=0.9977 acc_corrupt=0.5138 loss_corrupt=1.3025 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4007 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6324 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.7476 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8427 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6993 out_g_norm=1.2530 loss_all=1.4144 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=6.8s lr=2.000000e-03 loss=1.2849 loss_recon=1.2849 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5196 corrupt_frac=0.9977 acc_corrupt=0.5186 loss_corrupt=1.2849 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4090 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6338 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7528 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8412 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6953 out_g_norm=1.2064 acc_corrupt_t_0p8_1p0=0.9179 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1575 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=1750 epoch=1750/2000 epoch_step=1/1 micro_steps=1750 elapsed=6.8s lr=2.000000e-03 loss=1.2846 loss_recon=1.2846 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5192 corrupt_frac=0.9977 acc_corrupt=0.5182 loss_corrupt=1.2846 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.4069 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6334 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7514 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6830 out_g_norm=1.1992 acc_corrupt_t_0p6_0p8=0.8436 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9225 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2351 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=6.7s lr=2.000000e-03 loss=1.2555 loss_recon=1.2555 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5232 corrupt_frac=0.9977 acc_corrupt=0.5222 loss_corrupt=1.2555 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4130 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6357 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7511 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8439 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6717 out_g_norm=1.0249 acc_corrupt_t_0p8_1p0=0.9325 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2648 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=1850 epoch=1850/2000 epoch_step=1/1 micro_steps=1850 elapsed=6.7s lr=2.000000e-03 loss=1.2553 loss_recon=1.2553 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5221 corrupt_frac=0.9977 acc_corrupt=0.5212 loss_corrupt=1.2553 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.4130 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6353 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7527 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8436 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6649 out_g_norm=0.9855 loss_all=1.2214 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=6.6s lr=2.000000e-03 loss=1.2071 loss_recon=1.2071 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5331 corrupt_frac=0.9977 acc_corrupt=0.5322 loss_corrupt=1.2071 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4269 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6361 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7535 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8456 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6530 out_g_norm=1.1090 acc_corrupt_t_0p8_1p0=0.9086 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1394 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=1950 epoch=1950/2000 epoch_step=1/1 micro_steps=1950 elapsed=6.6s lr=2.000000e-03 loss=1.2169 loss_recon=1.2169 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5308 corrupt_frac=0.9977 acc_corrupt=0.5298 loss_corrupt=1.2169 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.4224 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6374 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7548 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6475 out_g_norm=0.9458 acc_corrupt_t_0p6_0p8=0.8471 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9226 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1710 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=6.6s lr=2.000000e-03 loss=1.1879 loss_recon=1.1879 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5333 corrupt_frac=0.9977 acc_corrupt=0.5323 loss_corrupt=1.1879 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4307 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6391 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7526 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6525 out_g_norm=0.8181 acc_corrupt_t_0p6_0p8=0.8494 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9176 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1480 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=2050 epoch=2050/2500 epoch_step=1/1 micro_steps=2050 elapsed=7.5s lr=2.000000e-03 loss=1.2013 loss_recon=1.2013 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5323 corrupt_frac=0.9977 acc_corrupt=0.5313 loss_corrupt=1.2013 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.4305 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6389 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.7535 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8465 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6565 out_g_norm=0.9416 acc_corrupt_t_0p8_1p0=0.9335 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1559 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=2100 epoch=2100/2500 epoch_step=1/1 micro_steps=2100 elapsed=6.5s lr=2.000000e-03 loss=1.1773 loss_recon=1.1773 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5370 corrupt_frac=0.9977 acc_corrupt=0.5361 loss_corrupt=1.1773 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4330 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6390 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.7545 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8422 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6524 out_g_norm=0.9270 acc_corrupt_t_0p8_1p0=0.9406 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2919 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=2150 epoch=2150/2500 epoch_step=1/1 micro_steps=2150 elapsed=6.5s lr=2.000000e-03 loss=1.1843 loss_recon=1.1843 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5351 corrupt_frac=0.9977 acc_corrupt=0.5342 loss_corrupt=1.1843 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4321 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6389 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.7541 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8474 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6558 out_g_norm=0.8821 loss_all=1.2592 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=2200 epoch=2200/2500 epoch_step=1/1 micro_steps=2200 elapsed=6.5s lr=2.000000e-03 loss=1.1592 loss_recon=1.1592 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5402 corrupt_frac=0.9977 acc_corrupt=0.5392 loss_corrupt=1.1592 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4408 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6404 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7585 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8476 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6522 out_g_norm=0.8903 acc_corrupt_t_0p8_1p0=0.9267 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0199 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=2250 epoch=2250/2500 epoch_step=1/1 micro_steps=2250 elapsed=6.5s lr=2.000000e-03 loss=1.1659 loss_recon=1.1659 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5391 corrupt_frac=0.9977 acc_corrupt=0.5381 loss_corrupt=1.1659 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.4377 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6398 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7567 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6528 out_g_norm=0.8228 acc_corrupt_t_0p6_0p8=0.8463 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9265 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0956 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=2300 epoch=2300/2500 epoch_step=1/1 micro_steps=2300 elapsed=6.5s lr=2.000000e-03 loss=1.1506 loss_recon=1.1506 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5417 corrupt_frac=0.9977 acc_corrupt=0.5408 loss_corrupt=1.1506 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4425 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6405 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7560 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8495 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6415 out_g_norm=0.8221 acc_corrupt_t_0p8_1p0=0.9344 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1599 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=2350 epoch=2350/2500 epoch_step=1/1 micro_steps=2350 elapsed=6.5s lr=2.000000e-03 loss=1.1642 loss_recon=1.1642 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5384 corrupt_frac=0.9977 acc_corrupt=0.5374 loss_corrupt=1.1642 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.4383 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6400 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7579 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8493 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6356 out_g_norm=0.7092 loss_all=1.1202 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=2400 epoch=2400/2500 epoch_step=1/1 micro_steps=2400 elapsed=6.5s lr=2.000000e-03 loss=1.1125 loss_recon=1.1125 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5497 corrupt_frac=0.9977 acc_corrupt=0.5488 loss_corrupt=1.1125 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4530 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6410 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7590 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8505 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6307 out_g_norm=0.8863 acc_corrupt_t_0p8_1p0=0.9183 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0588 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=2450 epoch=2450/2500 epoch_step=1/1 micro_steps=2450 elapsed=6.5s lr=2.000000e-03 loss=1.1299 loss_recon=1.1299 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5464 corrupt_frac=0.9977 acc_corrupt=0.5455 loss_corrupt=1.1299 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.4476 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6413 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7589 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6331 out_g_norm=0.8487 acc_corrupt_t_0p6_0p8=0.8531 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9226 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0615 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=2500 epoch=2500/2500 epoch_step=1/1 micro_steps=2500 elapsed=6.5s lr=2.000000e-03 loss=1.1263 loss_recon=1.1263 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5453 corrupt_frac=0.9977 acc_corrupt=0.5444 loss_corrupt=1.1263 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4502 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6412 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7568 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6342 out_g_norm=0.8608 acc_corrupt_t_0p6_0p8=0.8520 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9098 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1196 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=2501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=2550 epoch=2550/3000 epoch_step=1/1 micro_steps=2550 elapsed=7.4s lr=2.000000e-03 loss=1.1229 loss_recon=1.1229 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5460 corrupt_frac=0.9977 acc_corrupt=0.5450 loss_corrupt=1.1229 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.4516 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6426 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.7579 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8517 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6370 out_g_norm=0.7539 acc_corrupt_t_0p8_1p0=0.9394 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1046 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=6.8s lr=2.000000e-03 loss=1.0931 loss_recon=1.0931 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5521 corrupt_frac=0.9977 acc_corrupt=0.5511 loss_corrupt=1.0931 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4574 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6423 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.7586 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8453 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6408 out_g_norm=0.6935 acc_corrupt_t_0p8_1p0=0.9439 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3136 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=2650 epoch=2650/3000 epoch_step=1/1 micro_steps=2650 elapsed=6.8s lr=2.000000e-03 loss=1.0936 loss_recon=1.0936 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5519 corrupt_frac=0.9977 acc_corrupt=0.5510 loss_corrupt=1.0936 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4595 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6423 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.7581 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8503 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6389 out_g_norm=0.6920 loss_all=1.1896 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=6.8s lr=2.000000e-03 loss=1.0975 loss_recon=1.0975 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5518 corrupt_frac=0.9977 acc_corrupt=0.5509 loss_corrupt=1.0975 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4593 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6432 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7618 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8489 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6346 out_g_norm=0.7303 acc_corrupt_t_0p8_1p0=0.9189 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0518 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=2750 epoch=2750/3000 epoch_step=1/1 micro_steps=2750 elapsed=6.8s lr=2.000000e-03 loss=1.0776 loss_recon=1.0776 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5539 corrupt_frac=0.9977 acc_corrupt=0.5530 loss_corrupt=1.0776 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.4621 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6426 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7594 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6364 out_g_norm=0.7613 acc_corrupt_t_0p6_0p8=0.8515 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9324 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0531 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=6.8s lr=2.000000e-03 loss=1.0766 loss_recon=1.0766 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5552 corrupt_frac=0.9977 acc_corrupt=0.5543 loss_corrupt=1.0766 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4646 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6433 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7578 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8512 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6348 out_g_norm=0.7187 acc_corrupt_t_0p8_1p0=0.9315 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0794 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=2850 epoch=2850/3000 epoch_step=1/1 micro_steps=2850 elapsed=6.8s lr=2.000000e-03 loss=1.0786 loss_recon=1.0786 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5528 corrupt_frac=0.9977 acc_corrupt=0.5519 loss_corrupt=1.0786 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.4626 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6419 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7596 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8510 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6309 out_g_norm=0.7144 loss_all=1.0556 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=6.8s lr=2.000000e-03 loss=1.0319 loss_recon=1.0319 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5640 corrupt_frac=0.9977 acc_corrupt=0.5631 loss_corrupt=1.0319 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4774 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6431 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7607 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8513 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6334 out_g_norm=0.7229 acc_corrupt_t_0p8_1p0=0.9271 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0304 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=2950 epoch=2950/3000 epoch_step=1/1 micro_steps=2950 elapsed=6.6s lr=2.000000e-03 loss=1.0729 loss_recon=1.0729 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5570 corrupt_frac=0.9977 acc_corrupt=0.5561 loss_corrupt=1.0729 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.4650 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6434 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7613 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6327 out_g_norm=0.6642 acc_corrupt_t_0p6_0p8=0.8556 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9212 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0380 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=6.6s lr=2.000000e-03 loss=1.0530 loss_recon=1.0530 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5584 corrupt_frac=0.9977 acc_corrupt=0.5575 loss_corrupt=1.0530 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4715 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6437 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7591 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6340 out_g_norm=0.6235 acc_corrupt_t_0p6_0p8=0.8548 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9235 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0724 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=3050 epoch=3050/3500 epoch_step=1/1 micro_steps=3050 elapsed=7.6s lr=2.000000e-03 loss=1.0513 loss_recon=1.0513 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5584 corrupt_frac=0.9977 acc_corrupt=0.5575 loss_corrupt=1.0513 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.4721 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6446 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.7599 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8549 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6378 out_g_norm=0.6483 acc_corrupt_t_0p8_1p0=0.9350 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0505 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=3100 epoch=3100/3500 epoch_step=1/1 micro_steps=3100 elapsed=6.6s lr=2.000000e-03 loss=1.0321 loss_recon=1.0321 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5628 corrupt_frac=0.9977 acc_corrupt=0.5619 loss_corrupt=1.0321 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4751 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6447 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.7601 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8493 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6407 out_g_norm=0.5887 acc_corrupt_t_0p8_1p0=0.9452 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1816 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=3150 epoch=3150/3500 epoch_step=1/1 micro_steps=3150 elapsed=6.8s lr=2.000000e-03 loss=1.0297 loss_recon=1.0297 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5624 corrupt_frac=0.9977 acc_corrupt=0.5615 loss_corrupt=1.0297 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4762 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6449 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.7607 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8543 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6409 out_g_norm=0.5841 loss_all=1.1430 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=3200 epoch=3200/3500 epoch_step=1/1 micro_steps=3200 elapsed=6.8s lr=2.000000e-03 loss=1.0391 loss_recon=1.0391 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5617 corrupt_frac=0.9977 acc_corrupt=0.5607 loss_corrupt=1.0391 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4751 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6453 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7642 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8547 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6401 out_g_norm=0.6204 acc_corrupt_t_0p8_1p0=0.9247 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0135 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=3250 epoch=3250/3500 epoch_step=1/1 micro_steps=3250 elapsed=6.8s lr=2.000000e-03 loss=1.0144 loss_recon=1.0144 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5655 corrupt_frac=0.9977 acc_corrupt=0.5646 loss_corrupt=1.0144 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.4811 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6445 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7634 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6346 out_g_norm=0.6288 acc_corrupt_t_0p6_0p8=0.8530 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9284 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0060 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=3300 epoch=3300/3500 epoch_step=1/1 micro_steps=3300 elapsed=6.8s lr=2.000000e-03 loss=1.0406 loss_recon=1.0406 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5613 corrupt_frac=0.9977 acc_corrupt=0.5604 loss_corrupt=1.0406 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4745 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6445 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7605 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8534 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6311 out_g_norm=0.6854 acc_corrupt_t_0p8_1p0=0.9305 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9977 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=3350 epoch=3350/3500 epoch_step=1/1 micro_steps=3350 elapsed=6.8s lr=2.000000e-03 loss=1.0222 loss_recon=1.0222 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5637 corrupt_frac=0.9977 acc_corrupt=0.5628 loss_corrupt=1.0222 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.4800 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6446 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7637 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8556 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6284 out_g_norm=0.5560 loss_all=0.9896 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=3400 epoch=3400/3500 epoch_step=1/1 micro_steps=3400 elapsed=6.8s lr=2.000000e-03 loss=0.9952 loss_recon=0.9952 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5698 corrupt_frac=0.9977 acc_corrupt=0.5689 loss_corrupt=0.9952 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4857 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6455 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7651 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8558 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6302 out_g_norm=0.5724 acc_corrupt_t_0p8_1p0=0.9276 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9600 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=3450 epoch=3450/3500 epoch_step=1/1 micro_steps=3450 elapsed=6.7s lr=2.000000e-03 loss=1.0253 loss_recon=1.0253 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5651 corrupt_frac=0.9977 acc_corrupt=0.5641 loss_corrupt=1.0253 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.4773 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6462 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.7650 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6255 out_g_norm=0.6012 acc_corrupt_t_0p6_0p8=0.8592 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9324 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9762 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=3500 epoch=3500/3500 epoch_step=1/1 micro_steps=3500 elapsed=6.7s lr=2.000000e-03 loss=1.0063 loss_recon=1.0063 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5676 corrupt_frac=0.9977 acc_corrupt=0.5667 loss_corrupt=1.0063 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4853 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6466 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7646 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6252 out_g_norm=0.5622 acc_corrupt_t_0p6_0p8=0.8594 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9167 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0630 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=3501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=3550 epoch=3550/4000 epoch_step=1/1 micro_steps=3550 elapsed=7.6s lr=2.000000e-03 loss=1.0228 loss_recon=1.0228 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5659 corrupt_frac=0.9977 acc_corrupt=0.5650 loss_corrupt=1.0228 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.4824 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6476 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.7672 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8605 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6258 out_g_norm=0.5701 acc_corrupt_t_0p8_1p0=0.9423 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0376 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=6.6s lr=2.000000e-03 loss=0.9899 loss_recon=0.9899 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5712 corrupt_frac=0.9977 acc_corrupt=0.5703 loss_corrupt=0.9899 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4854 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6495 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.7712 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8599 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6248 out_g_norm=0.5156 acc_corrupt_t_0p8_1p0=0.9488 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1609 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=3650 epoch=3650/4000 epoch_step=1/1 micro_steps=3650 elapsed=6.7s lr=2.000000e-03 loss=0.9944 loss_recon=0.9944 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5718 corrupt_frac=0.9977 acc_corrupt=0.5709 loss_corrupt=0.9944 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4866 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6514 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.7760 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.8704 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6336 out_g_norm=0.5215 loss_all=1.0478 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=6.8s lr=2.000000e-03 loss=0.9804 loss_recon=0.9804 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5751 corrupt_frac=0.9977 acc_corrupt=0.5742 loss_corrupt=0.9804 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4907 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6540 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.7835 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.8738 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6336 out_g_norm=0.5383 acc_corrupt_t_0p8_1p0=0.9404 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9459 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=3750 epoch=3750/4000 epoch_step=1/1 micro_steps=3750 elapsed=7.0s lr=2.000000e-03 loss=0.9742 loss_recon=0.9742 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5773 corrupt_frac=0.9977 acc_corrupt=0.5764 loss_corrupt=0.9742 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.4917 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6551 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.7873 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6295 out_g_norm=0.5837 acc_corrupt_t_0p6_0p8=0.8817 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9539 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0013 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=6.9s lr=2.000000e-03 loss=0.9790 loss_recon=0.9790 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5769 corrupt_frac=0.9977 acc_corrupt=0.5760 loss_corrupt=0.9790 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4894 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6578 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.7903 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.8861 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6275 out_g_norm=0.5622 acc_corrupt_t_0p8_1p0=0.9648 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9725 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=3850 epoch=3850/4000 epoch_step=1/1 micro_steps=3850 elapsed=6.9s lr=2.000000e-03 loss=0.9713 loss_recon=0.9713 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5775 corrupt_frac=0.9977 acc_corrupt=0.5766 loss_corrupt=0.9713 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.4908 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6586 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.7961 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.8908 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6249 out_g_norm=0.5280 loss_all=0.9700 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=7.0s lr=2.000000e-03 loss=0.9509 loss_recon=0.9509 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5846 corrupt_frac=0.9977 acc_corrupt=0.5838 loss_corrupt=0.9509 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.4968 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6610 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.7994 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.8958 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6274 out_g_norm=0.5444 acc_corrupt_t_0p8_1p0=0.9540 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9394 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=3950 epoch=3950/4000 epoch_step=1/1 micro_steps=3950 elapsed=6.8s lr=2.000000e-03 loss=0.9593 loss_recon=0.9593 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5826 corrupt_frac=0.9977 acc_corrupt=0.5817 loss_corrupt=0.9593 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.4924 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6626 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8022 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6200 out_g_norm=0.5365 acc_corrupt_t_0p6_0p8=0.8992 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9643 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9136 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=6.7s lr=2.000000e-03 loss=0.9735 loss_recon=0.9735 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5797 corrupt_frac=0.9977 acc_corrupt=0.5788 loss_corrupt=0.9735 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4915 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6623 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.7998 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6170 out_g_norm=0.5837 acc_corrupt_t_0p6_0p8=0.8990 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9490 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8982 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=4050 epoch=4050/4500 epoch_step=1/1 micro_steps=4050 elapsed=7.6s lr=2.000000e-03 loss=0.9731 loss_recon=0.9731 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5796 corrupt_frac=0.9977 acc_corrupt=0.5787 loss_corrupt=0.9731 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.4909 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6645 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8019 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9006 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6179 out_g_norm=0.5941 acc_corrupt_t_0p8_1p0=0.9731 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9463 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=4100 epoch=4100/4500 epoch_step=1/1 micro_steps=4100 elapsed=6.6s lr=2.000000e-03 loss=0.9457 loss_recon=0.9457 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5858 corrupt_frac=0.9977 acc_corrupt=0.5849 loss_corrupt=0.9457 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.4974 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6642 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8022 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.8965 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6228 out_g_norm=0.6041 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1154 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=4150 epoch=4150/4500 epoch_step=1/1 micro_steps=4150 elapsed=6.6s lr=2.000000e-03 loss=0.9520 loss_recon=0.9520 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5844 corrupt_frac=0.9977 acc_corrupt=0.5835 loss_corrupt=0.9520 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.4957 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6657 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8032 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9020 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6195 out_g_norm=0.4815 loss_all=0.9657 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=4200 epoch=4200/4500 epoch_step=1/1 micro_steps=4200 elapsed=6.5s lr=2.000000e-03 loss=0.9494 loss_recon=0.9494 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5858 corrupt_frac=0.9977 acc_corrupt=0.5849 loss_corrupt=0.9494 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.4982 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6663 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8078 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9020 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6334 out_g_norm=0.5348 acc_corrupt_t_0p8_1p0=0.9658 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9124 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=4250 epoch=4250/4500 epoch_step=1/1 micro_steps=4250 elapsed=6.6s lr=2.000000e-03 loss=0.9334 loss_recon=0.9334 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5887 corrupt_frac=0.9977 acc_corrupt=0.5878 loss_corrupt=0.9334 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5017 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6665 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8069 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6385 out_g_norm=0.5175 acc_corrupt_t_0p6_0p8=0.9035 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9241 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=4300 epoch=4300/4500 epoch_step=1/1 micro_steps=4300 elapsed=6.7s lr=2.000000e-03 loss=0.9464 loss_recon=0.9464 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5862 corrupt_frac=0.9977 acc_corrupt=0.5853 loss_corrupt=0.9464 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4982 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6666 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8051 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9035 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6316 out_g_norm=0.4943 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9645 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=4350 epoch=4350/4500 epoch_step=1/1 micro_steps=4350 elapsed=6.6s lr=2.000000e-03 loss=0.9412 loss_recon=0.9412 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5866 corrupt_frac=0.9977 acc_corrupt=0.5858 loss_corrupt=0.9412 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5011 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6657 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8074 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9047 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6299 out_g_norm=0.4762 loss_all=0.8833 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=4400 epoch=4400/4500 epoch_step=1/1 micro_steps=4400 elapsed=6.6s lr=2.000000e-03 loss=0.9181 loss_recon=0.9181 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5915 corrupt_frac=0.9977 acc_corrupt=0.5907 loss_corrupt=0.9181 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5042 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6667 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8085 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9058 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6280 out_g_norm=0.4954 acc_corrupt_t_0p8_1p0=0.9648 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8815 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=4450 epoch=4450/4500 epoch_step=1/1 micro_steps=4450 elapsed=6.6s lr=2.000000e-03 loss=0.9321 loss_recon=0.9321 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5900 corrupt_frac=0.9977 acc_corrupt=0.5892 loss_corrupt=0.9321 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5012 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6682 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8092 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6276 out_g_norm=0.4932 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9181 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=4500 epoch=4500/4500 epoch_step=1/1 micro_steps=4500 elapsed=6.6s lr=2.000000e-03 loss=0.9432 loss_recon=0.9432 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5863 corrupt_frac=0.9977 acc_corrupt=0.5854 loss_corrupt=0.9432 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.4994 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6671 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8057 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6326 out_g_norm=0.4797 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9529 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9453 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=4501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=4550 epoch=4550/5000 epoch_step=1/1 micro_steps=4550 elapsed=7.6s lr=2.000000e-03 loss=0.9221 loss_recon=0.9221 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5897 corrupt_frac=0.9977 acc_corrupt=0.5888 loss_corrupt=0.9221 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5046 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6699 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8069 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9078 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6312 out_g_norm=0.4949 acc_corrupt_t_0p8_1p0=0.9790 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9246 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=6.7s lr=2.000000e-03 loss=0.9064 loss_recon=0.9064 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5933 corrupt_frac=0.9977 acc_corrupt=0.5925 loss_corrupt=0.9064 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5070 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6692 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8074 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9010 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6371 out_g_norm=0.5234 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0162 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=4650 epoch=4650/5000 epoch_step=1/1 micro_steps=4650 elapsed=6.6s lr=2.000000e-03 loss=0.9353 loss_recon=0.9353 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5884 corrupt_frac=0.9977 acc_corrupt=0.5875 loss_corrupt=0.9353 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5002 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6692 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8068 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9059 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6324 out_g_norm=0.5024 loss_all=0.9988 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=6.6s lr=2.000000e-03 loss=0.9114 loss_recon=0.9114 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5918 corrupt_frac=0.9977 acc_corrupt=0.5909 loss_corrupt=0.9114 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5055 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6708 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9059 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6375 out_g_norm=0.4908 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8739 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=4750 epoch=4750/5000 epoch_step=1/1 micro_steps=4750 elapsed=6.6s lr=2.000000e-03 loss=0.9100 loss_recon=0.9100 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5930 corrupt_frac=0.9977 acc_corrupt=0.5922 loss_corrupt=0.9100 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5070 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6698 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8096 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6504 out_g_norm=0.5299 acc_corrupt_t_0p6_0p8=0.9070 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8959 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=6.6s lr=2.000000e-03 loss=0.9219 loss_recon=0.9219 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5905 corrupt_frac=0.9977 acc_corrupt=0.5896 loss_corrupt=0.9219 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5035 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6698 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8073 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9080 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6453 out_g_norm=0.5534 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9190 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=4850 epoch=4850/5000 epoch_step=1/1 micro_steps=4850 elapsed=6.6s lr=2.000000e-03 loss=0.9279 loss_recon=0.9279 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5894 corrupt_frac=0.9977 acc_corrupt=0.5886 loss_corrupt=0.9279 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5045 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6680 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8086 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9065 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6329 out_g_norm=0.5126 loss_all=0.8873 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=6.6s lr=2.000000e-03 loss=0.8864 loss_recon=0.8864 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5975 corrupt_frac=0.9977 acc_corrupt=0.5967 loss_corrupt=0.8864 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5126 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6700 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6305 out_g_norm=0.4892 acc_corrupt_t_0p8_1p0=0.9721 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8319 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=4950 epoch=4950/5000 epoch_step=1/1 micro_steps=4950 elapsed=6.6s lr=2.000000e-03 loss=0.9032 loss_recon=0.9032 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5951 corrupt_frac=0.9977 acc_corrupt=0.5942 loss_corrupt=0.9032 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5082 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6707 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6299 out_g_norm=0.5142 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9691 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9550 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=6.6s lr=2.000000e-03 loss=0.9170 loss_recon=0.9170 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5912 corrupt_frac=0.9977 acc_corrupt=0.5903 loss_corrupt=0.9170 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5063 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6696 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8078 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6378 out_g_norm=0.5036 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9647 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9151 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=5050 epoch=5050/5500 epoch_step=1/1 micro_steps=5050 elapsed=8.1s lr=2.000000e-03 loss=0.9082 loss_recon=0.9082 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5930 corrupt_frac=0.9977 acc_corrupt=0.5921 loss_corrupt=0.9082 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5092 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8087 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9075 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6356 out_g_norm=0.4901 acc_corrupt_t_0p8_1p0=0.9760 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8892 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=5100 epoch=5100/5500 epoch_step=1/1 micro_steps=5100 elapsed=6.8s lr=2.000000e-03 loss=0.8866 loss_recon=0.8866 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5968 corrupt_frac=0.9977 acc_corrupt=0.5959 loss_corrupt=0.8866 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5117 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8091 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9039 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6388 out_g_norm=0.4709 acc_corrupt_t_0p8_1p0=0.9811 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9177 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=5150 epoch=5150/5500 epoch_step=1/1 micro_steps=5150 elapsed=6.7s lr=2.000000e-03 loss=0.8754 loss_recon=0.8754 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5981 corrupt_frac=0.9977 acc_corrupt=0.5972 loss_corrupt=0.8754 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5150 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8091 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6511 out_g_norm=0.4843 loss_all=1.0208 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=5200 epoch=5200/5500 epoch_step=1/1 micro_steps=5200 elapsed=6.7s lr=2.000000e-03 loss=0.8867 loss_recon=0.8867 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5959 corrupt_frac=0.9977 acc_corrupt=0.5950 loss_corrupt=0.8867 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5114 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6627 out_g_norm=0.5131 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8427 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=5250 epoch=5250/5500 epoch_step=1/1 micro_steps=5250 elapsed=6.9s lr=2.000000e-03 loss=0.8838 loss_recon=0.8838 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5975 corrupt_frac=0.9977 acc_corrupt=0.5966 loss_corrupt=0.8838 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5137 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6715 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6625 out_g_norm=0.4162 acc_corrupt_t_0p6_0p8=0.9077 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9784 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8909 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=5300 epoch=5300/5500 epoch_step=1/1 micro_steps=5300 elapsed=7.0s lr=2.000000e-03 loss=0.8958 loss_recon=0.8958 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5960 corrupt_frac=0.9977 acc_corrupt=0.5951 loss_corrupt=0.8958 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5117 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6720 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8088 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9076 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6550 out_g_norm=0.4080 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9169 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=5350 epoch=5350/5500 epoch_step=1/1 micro_steps=5350 elapsed=6.9s lr=2.000000e-03 loss=0.9067 loss_recon=0.9067 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5928 corrupt_frac=0.9977 acc_corrupt=0.5919 loss_corrupt=0.9067 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5085 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6706 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6444 out_g_norm=0.4937 loss_all=0.9004 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=5400 epoch=5400/5500 epoch_step=1/1 micro_steps=5400 elapsed=6.7s lr=2.000000e-03 loss=0.8788 loss_recon=0.8788 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5989 corrupt_frac=0.9977 acc_corrupt=0.5981 loss_corrupt=0.8788 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5142 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6712 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6438 out_g_norm=0.4187 acc_corrupt_t_0p8_1p0=0.9731 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9019 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=5450 epoch=5450/5500 epoch_step=1/1 micro_steps=5450 elapsed=6.7s lr=2.000000e-03 loss=0.8879 loss_recon=0.8879 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5975 corrupt_frac=0.9977 acc_corrupt=0.5967 loss_corrupt=0.8879 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5116 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6441 out_g_norm=0.4617 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8929 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=5500 epoch=5500/5500 epoch_step=1/1 micro_steps=5500 elapsed=6.7s lr=2.000000e-03 loss=0.9171 loss_recon=0.9171 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5911 corrupt_frac=0.9977 acc_corrupt=0.5902 loss_corrupt=0.9171 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5053 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6706 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8085 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6382 out_g_norm=0.4508 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9696 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8469 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=5501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=5550 epoch=5550/6000 epoch_step=1/1 micro_steps=5550 elapsed=7.5s lr=2.000000e-03 loss=0.8883 loss_recon=0.8883 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5962 corrupt_frac=0.9977 acc_corrupt=0.5954 loss_corrupt=0.8883 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5137 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8098 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6352 out_g_norm=0.3990 acc_corrupt_t_0p8_1p0=0.9682 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8343 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=6.7s lr=2.000000e-03 loss=0.8676 loss_recon=0.8676 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6006 corrupt_frac=0.9977 acc_corrupt=0.5997 loss_corrupt=0.8676 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5176 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8095 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9036 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6319 out_g_norm=0.4001 acc_corrupt_t_0p8_1p0=0.9794 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9608 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=5650 epoch=5650/6000 epoch_step=1/1 micro_steps=5650 elapsed=6.8s lr=2.000000e-03 loss=0.8753 loss_recon=0.8753 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5987 corrupt_frac=0.9977 acc_corrupt=0.5979 loss_corrupt=0.8753 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5158 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6732 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8091 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6384 out_g_norm=0.4353 loss_all=0.9736 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=6.8s lr=2.000000e-03 loss=0.8826 loss_recon=0.8826 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5973 corrupt_frac=0.9977 acc_corrupt=0.5965 loss_corrupt=0.8826 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5135 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6383 out_g_norm=0.4758 acc_corrupt_t_0p8_1p0=0.9677 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8974 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=5750 epoch=5750/6000 epoch_step=1/1 micro_steps=5750 elapsed=6.8s lr=2.000000e-03 loss=0.8843 loss_recon=0.8843 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5974 corrupt_frac=0.9977 acc_corrupt=0.5966 loss_corrupt=0.8843 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5135 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6716 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6373 out_g_norm=0.4172 acc_corrupt_t_0p6_0p8=0.9078 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9794 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9056 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=6.8s lr=2.000000e-03 loss=0.8767 loss_recon=0.8767 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5984 corrupt_frac=0.9977 acc_corrupt=0.5975 loss_corrupt=0.8767 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5156 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8089 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6378 out_g_norm=0.4502 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8176 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=5850 epoch=5850/6000 epoch_step=1/1 micro_steps=5850 elapsed=6.8s lr=2.000000e-03 loss=0.8886 loss_recon=0.8886 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5957 corrupt_frac=0.9977 acc_corrupt=0.5948 loss_corrupt=0.8886 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5135 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6709 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6323 out_g_norm=0.4412 loss_all=0.8985 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=6.8s lr=2.000000e-03 loss=0.8536 loss_recon=0.8536 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6037 corrupt_frac=0.9977 acc_corrupt=0.6028 loss_corrupt=0.8536 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5217 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6729 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6340 out_g_norm=0.3975 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8076 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=5950 epoch=5950/6000 epoch_step=1/1 micro_steps=5950 elapsed=6.8s lr=2.000000e-03 loss=0.8895 loss_recon=0.8895 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5970 corrupt_frac=0.9977 acc_corrupt=0.5961 loss_corrupt=0.8895 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5103 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6342 out_g_norm=0.4047 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9706 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8697 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=6.8s lr=2.000000e-03 loss=0.8904 loss_recon=0.8904 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5951 corrupt_frac=0.9977 acc_corrupt=0.5943 loss_corrupt=0.8904 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5121 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6713 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8085 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6316 out_g_norm=0.4572 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9696 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8922 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=6050 epoch=6050/6500 epoch_step=1/1 micro_steps=6050 elapsed=7.9s lr=2.000000e-03 loss=0.8868 loss_recon=0.8868 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5967 corrupt_frac=0.9977 acc_corrupt=0.5958 loss_corrupt=0.8868 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5144 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8092 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9082 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6244 out_g_norm=0.4018 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9308 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=6100 epoch=6100/6500 epoch_step=1/1 micro_steps=6100 elapsed=6.6s lr=2.000000e-03 loss=0.8779 loss_recon=0.8779 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5981 corrupt_frac=0.9977 acc_corrupt=0.5973 loss_corrupt=0.8779 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5135 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6721 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8089 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9034 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6265 out_g_norm=0.4321 acc_corrupt_t_0p8_1p0=0.9821 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9360 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=6150 epoch=6150/6500 epoch_step=1/1 micro_steps=6150 elapsed=6.5s lr=2.000000e-03 loss=0.8684 loss_recon=0.8684 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6000 corrupt_frac=0.9977 acc_corrupt=0.5991 loss_corrupt=0.8684 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5182 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6730 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8094 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6202 out_g_norm=0.4297 loss_all=0.9246 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=6200 epoch=6200/6500 epoch_step=1/1 micro_steps=6200 elapsed=6.5s lr=2.000000e-03 loss=0.8641 loss_recon=0.8641 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6004 corrupt_frac=0.9977 acc_corrupt=0.5995 loss_corrupt=0.8641 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6258 out_g_norm=0.4486 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8761 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=6250 epoch=6250/6500 epoch_step=1/1 micro_steps=6250 elapsed=6.5s lr=2.000000e-03 loss=0.8683 loss_recon=0.8683 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6001 corrupt_frac=0.9977 acc_corrupt=0.5993 loss_corrupt=0.8683 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5177 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6724 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6221 out_g_norm=0.4241 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8614 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=6300 epoch=6300/6500 epoch_step=1/1 micro_steps=6300 elapsed=6.6s lr=2.000000e-03 loss=0.8675 loss_recon=0.8675 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6006 corrupt_frac=0.9977 acc_corrupt=0.5997 loss_corrupt=0.8675 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5190 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9087 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6174 out_g_norm=0.5395 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8563 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=6350 epoch=6350/6500 epoch_step=1/1 micro_steps=6350 elapsed=6.5s lr=2.000000e-03 loss=0.8838 loss_recon=0.8838 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5962 corrupt_frac=0.9977 acc_corrupt=0.5954 loss_corrupt=0.8838 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5139 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6716 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6103 out_g_norm=0.4403 loss_all=0.8426 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=6400 epoch=6400/6500 epoch_step=1/1 micro_steps=6400 elapsed=6.5s lr=2.000000e-03 loss=0.8616 loss_recon=0.8616 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6022 corrupt_frac=0.9977 acc_corrupt=0.6013 loss_corrupt=0.8616 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5190 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6728 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6053 out_g_norm=0.4553 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7915 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=6450 epoch=6450/6500 epoch_step=1/1 micro_steps=6450 elapsed=6.5s lr=2.000000e-03 loss=0.8595 loss_recon=0.8595 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6018 corrupt_frac=0.9977 acc_corrupt=0.6009 loss_corrupt=0.8595 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5184 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6131 out_g_norm=0.3924 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9696 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8621 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=6500 epoch=6500/6500 epoch_step=1/1 micro_steps=6500 elapsed=6.5s lr=2.000000e-03 loss=0.8586 loss_recon=0.8586 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6010 corrupt_frac=0.9977 acc_corrupt=0.6001 loss_corrupt=0.8586 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5208 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6735 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6238 out_g_norm=0.4064 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9745 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8028 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=6501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=6550 epoch=6550/7000 epoch_step=1/1 micro_steps=6550 elapsed=7.5s lr=2.000000e-03 loss=0.8567 loss_recon=0.8567 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6019 corrupt_frac=0.9977 acc_corrupt=0.6011 loss_corrupt=0.8567 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5224 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8103 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6276 out_g_norm=0.4390 acc_corrupt_t_0p8_1p0=0.9741 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8607 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=6600 epoch=6600/7000 epoch_step=1/1 micro_steps=6600 elapsed=6.7s lr=2.000000e-03 loss=0.8338 loss_recon=0.8338 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6056 corrupt_frac=0.9977 acc_corrupt=0.6048 loss_corrupt=0.8338 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5251 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9050 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6356 out_g_norm=0.3841 acc_corrupt_t_0p8_1p0=0.9808 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9117 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=6650 epoch=6650/7000 epoch_step=1/1 micro_steps=6650 elapsed=6.6s lr=2.000000e-03 loss=0.8523 loss_recon=0.8523 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6024 corrupt_frac=0.9977 acc_corrupt=0.6015 loss_corrupt=0.8523 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5215 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6744 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8097 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9087 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6380 out_g_norm=0.3961 loss_all=0.9698 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=6700 epoch=6700/7000 epoch_step=1/1 micro_steps=6700 elapsed=6.6s lr=2.000000e-03 loss=0.8613 loss_recon=0.8613 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6006 corrupt_frac=0.9977 acc_corrupt=0.5997 loss_corrupt=0.8613 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5184 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6337 out_g_norm=0.4075 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7960 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=6750 epoch=6750/7000 epoch_step=1/1 micro_steps=6750 elapsed=6.6s lr=2.000000e-03 loss=0.8562 loss_recon=0.8562 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6025 corrupt_frac=0.9977 acc_corrupt=0.6016 loss_corrupt=0.8562 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5211 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6275 out_g_norm=0.3951 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8382 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=6800 epoch=6800/7000 epoch_step=1/1 micro_steps=6800 elapsed=6.6s lr=2.000000e-03 loss=0.8552 loss_recon=0.8552 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6025 corrupt_frac=0.9977 acc_corrupt=0.6016 loss_corrupt=0.8552 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5214 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8101 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6305 out_g_norm=0.3590 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8827 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=6850 epoch=6850/7000 epoch_step=1/1 micro_steps=6850 elapsed=6.6s lr=2.000000e-03 loss=0.8674 loss_recon=0.8674 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5994 corrupt_frac=0.9977 acc_corrupt=0.5985 loss_corrupt=0.8674 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5189 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6726 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6261 out_g_norm=0.3918 loss_all=0.8378 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=6900 epoch=6900/7000 epoch_step=1/1 micro_steps=6900 elapsed=6.6s lr=2.000000e-03 loss=0.8335 loss_recon=0.8335 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6065 corrupt_frac=0.9977 acc_corrupt=0.6056 loss_corrupt=0.8335 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5254 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6255 out_g_norm=0.3407 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7973 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=6950 epoch=6950/7000 epoch_step=1/1 micro_steps=6950 elapsed=6.6s lr=2.000000e-03 loss=0.8545 loss_recon=0.8545 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6031 corrupt_frac=0.9977 acc_corrupt=0.6022 loss_corrupt=0.8545 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5203 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6200 out_g_norm=0.3860 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9721 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8742 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=7000 epoch=7000/7000 epoch_step=1/1 micro_steps=7000 elapsed=6.6s lr=2.000000e-03 loss=0.8561 loss_recon=0.8561 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6014 corrupt_frac=0.9977 acc_corrupt=0.6006 loss_corrupt=0.8561 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5215 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6736 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6114 out_g_norm=0.4107 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9706 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8017 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=7001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=7050 epoch=7050/7500 epoch_step=1/1 micro_steps=7050 elapsed=7.5s lr=2.000000e-03 loss=0.8548 loss_recon=0.8548 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6015 corrupt_frac=0.9977 acc_corrupt=0.6006 loss_corrupt=0.8548 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5215 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6151 out_g_norm=0.4766 acc_corrupt_t_0p8_1p0=0.9780 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8340 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=7100 epoch=7100/7500 epoch_step=1/1 micro_steps=7100 elapsed=6.5s lr=2.000000e-03 loss=0.8491 loss_recon=0.8491 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6039 corrupt_frac=0.9977 acc_corrupt=0.6031 loss_corrupt=0.8491 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5223 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8097 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9039 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6183 out_g_norm=0.4386 acc_corrupt_t_0p8_1p0=0.9808 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9161 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=7150 epoch=7150/7500 epoch_step=1/1 micro_steps=7150 elapsed=6.5s lr=2.000000e-03 loss=0.8456 loss_recon=0.8456 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6032 corrupt_frac=0.9977 acc_corrupt=0.6023 loss_corrupt=0.8456 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5233 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8090 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6179 out_g_norm=0.4283 loss_all=0.9089 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=7200 epoch=7200/7500 epoch_step=1/1 micro_steps=7200 elapsed=6.5s lr=2.000000e-03 loss=0.8507 loss_recon=0.8507 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6026 corrupt_frac=0.9977 acc_corrupt=0.6018 loss_corrupt=0.8507 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5216 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8140 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6287 out_g_norm=0.4399 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8070 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=7250 epoch=7250/7500 epoch_step=1/1 micro_steps=7250 elapsed=6.5s lr=2.000000e-03 loss=0.8436 loss_recon=0.8436 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6044 corrupt_frac=0.9977 acc_corrupt=0.6036 loss_corrupt=0.8436 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5244 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6348 out_g_norm=0.3769 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7976 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=7300 epoch=7300/7500 epoch_step=1/1 micro_steps=7300 elapsed=6.6s lr=2.000000e-03 loss=0.8699 loss_recon=0.8699 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5999 corrupt_frac=0.9977 acc_corrupt=0.5990 loss_corrupt=0.8699 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5169 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6301 out_g_norm=0.4063 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8988 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=7350 epoch=7350/7500 epoch_step=1/1 micro_steps=7350 elapsed=6.6s lr=2.000000e-03 loss=0.8657 loss_recon=0.8657 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5990 corrupt_frac=0.9977 acc_corrupt=0.5982 loss_corrupt=0.8657 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5182 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6727 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6197 out_g_norm=0.4054 loss_all=0.8468 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=7400 epoch=7400/7500 epoch_step=1/1 micro_steps=7400 elapsed=6.6s lr=2.000000e-03 loss=0.8396 loss_recon=0.8396 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6060 corrupt_frac=0.9977 acc_corrupt=0.6052 loss_corrupt=0.8396 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5249 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6149 out_g_norm=0.4100 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7985 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=7450 epoch=7450/7500 epoch_step=1/1 micro_steps=7450 elapsed=6.6s lr=2.000000e-03 loss=0.8546 loss_recon=0.8546 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6031 corrupt_frac=0.9977 acc_corrupt=0.6022 loss_corrupt=0.8546 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5199 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6141 out_g_norm=0.4022 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8787 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=7500 epoch=7500/7500 epoch_step=1/1 micro_steps=7500 elapsed=6.5s lr=2.000000e-03 loss=0.8619 loss_recon=0.8619 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6005 corrupt_frac=0.9977 acc_corrupt=0.5996 loss_corrupt=0.8619 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5197 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6157 out_g_norm=0.3820 acc_corrupt_t_0p6_0p8=0.9124 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9725 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7784 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=7501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=7550 epoch=7550/8000 epoch_step=1/1 micro_steps=7550 elapsed=7.9s lr=2.000000e-03 loss=0.8446 loss_recon=0.8446 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6032 corrupt_frac=0.9977 acc_corrupt=0.6024 loss_corrupt=0.8446 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5246 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6137 out_g_norm=0.4072 acc_corrupt_t_0p8_1p0=0.9760 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8812 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=7600 epoch=7600/8000 epoch_step=1/1 micro_steps=7600 elapsed=6.7s lr=2.000000e-03 loss=0.8256 loss_recon=0.8256 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8256 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5278 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9041 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6156 out_g_norm=0.4123 acc_corrupt_t_0p8_1p0=0.9804 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8447 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=7650 epoch=7650/8000 epoch_step=1/1 micro_steps=7650 elapsed=6.7s lr=2.000000e-03 loss=0.8460 loss_recon=0.8460 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6029 corrupt_frac=0.9977 acc_corrupt=0.6021 loss_corrupt=0.8460 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5223 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8095 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6143 out_g_norm=0.3709 loss_all=0.8147 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=7700 epoch=7700/8000 epoch_step=1/1 micro_steps=7700 elapsed=6.6s lr=2.000000e-03 loss=0.8451 loss_recon=0.8451 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6032 corrupt_frac=0.9977 acc_corrupt=0.6023 loss_corrupt=0.8451 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5223 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6104 out_g_norm=0.4072 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7912 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=7750 epoch=7750/8000 epoch_step=1/1 micro_steps=7750 elapsed=6.6s lr=2.000000e-03 loss=0.8403 loss_recon=0.8403 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6044 corrupt_frac=0.9977 acc_corrupt=0.6036 loss_corrupt=0.8403 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5242 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6128 out_g_norm=0.4503 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8280 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=7800 epoch=7800/8000 epoch_step=1/1 micro_steps=7800 elapsed=6.6s lr=2.000000e-03 loss=0.8653 loss_recon=0.8653 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6009 corrupt_frac=0.9977 acc_corrupt=0.6000 loss_corrupt=0.8653 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5192 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6739 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9078 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6131 out_g_norm=0.5097 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8000 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=7850 epoch=7850/8000 epoch_step=1/1 micro_steps=7850 elapsed=6.6s lr=2.000000e-03 loss=0.8624 loss_recon=0.8624 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5997 corrupt_frac=0.9977 acc_corrupt=0.5988 loss_corrupt=0.8624 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5188 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6733 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9089 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6068 out_g_norm=0.3799 loss_all=0.8784 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=7900 epoch=7900/8000 epoch_step=1/1 micro_steps=7900 elapsed=6.6s lr=2.000000e-03 loss=0.8319 loss_recon=0.8319 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6066 corrupt_frac=0.9977 acc_corrupt=0.6058 loss_corrupt=0.8319 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5258 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6073 out_g_norm=0.3606 acc_corrupt_t_0p8_1p0=0.9692 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8249 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=7950 epoch=7950/8000 epoch_step=1/1 micro_steps=7950 elapsed=6.6s lr=2.000000e-03 loss=0.8350 loss_recon=0.8350 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6048 corrupt_frac=0.9977 acc_corrupt=0.6040 loss_corrupt=0.8350 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5233 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.6094 out_g_norm=0.4342 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9682 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8318 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=8000 epoch=8000/8000 epoch_step=1/1 micro_steps=8000 elapsed=6.7s lr=2.000000e-03 loss=0.8460 loss_recon=0.8460 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6028 corrupt_frac=0.9977 acc_corrupt=0.6019 loss_corrupt=0.8460 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5229 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6751 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.6083 out_g_norm=0.3450 acc_corrupt_t_0p6_0p8=0.9131 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9735 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7952 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=8001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=8050 epoch=8050/8500 epoch_step=1/1 micro_steps=8050 elapsed=7.6s lr=2.000000e-03 loss=0.8449 loss_recon=0.8449 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6026 corrupt_frac=0.9977 acc_corrupt=0.6017 loss_corrupt=0.8449 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5230 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6094 out_g_norm=0.3413 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8261 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=8100 epoch=8100/8500 epoch_step=1/1 micro_steps=8100 elapsed=6.6s lr=2.000000e-03 loss=0.8282 loss_recon=0.8282 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6062 corrupt_frac=0.9977 acc_corrupt=0.6053 loss_corrupt=0.8282 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5262 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6746 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9034 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.6114 out_g_norm=0.3610 acc_corrupt_t_0p8_1p0=0.9801 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8611 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=8150 epoch=8150/8500 epoch_step=1/1 micro_steps=8150 elapsed=6.7s lr=2.000000e-03 loss=0.8370 loss_recon=0.8370 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6046 corrupt_frac=0.9977 acc_corrupt=0.6037 loss_corrupt=0.8370 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5251 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8096 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6107 out_g_norm=0.3565 loss_all=0.8402 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=8200 epoch=8200/8500 epoch_step=1/1 micro_steps=8200 elapsed=6.7s lr=2.000000e-03 loss=0.8389 loss_recon=0.8389 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6039 corrupt_frac=0.9977 acc_corrupt=0.6030 loss_corrupt=0.8389 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5235 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6063 out_g_norm=0.3812 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7751 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=8250 epoch=8250/8500 epoch_step=1/1 micro_steps=8250 elapsed=6.6s lr=2.000000e-03 loss=0.8331 loss_recon=0.8331 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6059 corrupt_frac=0.9977 acc_corrupt=0.6051 loss_corrupt=0.8331 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5264 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6065 out_g_norm=0.3435 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8988 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=8300 epoch=8300/8500 epoch_step=1/1 micro_steps=8300 elapsed=6.6s lr=2.000000e-03 loss=0.8420 loss_recon=0.8420 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6039 corrupt_frac=0.9977 acc_corrupt=0.6031 loss_corrupt=0.8420 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5237 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9086 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6079 out_g_norm=0.3815 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8044 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=8350 epoch=8350/8500 epoch_step=1/1 micro_steps=8350 elapsed=6.6s lr=2.000000e-03 loss=0.8372 loss_recon=0.8372 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6037 corrupt_frac=0.9977 acc_corrupt=0.6028 loss_corrupt=0.8372 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5256 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.6017 out_g_norm=0.3535 loss_all=0.8181 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=8400 epoch=8400/8500 epoch_step=1/1 micro_steps=8400 elapsed=6.6s lr=2.000000e-03 loss=0.8340 loss_recon=0.8340 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6062 corrupt_frac=0.9977 acc_corrupt=0.6054 loss_corrupt=0.8340 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5252 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.6021 out_g_norm=0.3531 acc_corrupt_t_0p8_1p0=0.9667 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8390 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=8450 epoch=8450/8500 epoch_step=1/1 micro_steps=8450 elapsed=6.8s lr=2.000000e-03 loss=0.8430 loss_recon=0.8430 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6046 corrupt_frac=0.9977 acc_corrupt=0.6038 loss_corrupt=0.8430 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5229 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5936 out_g_norm=0.3777 acc_corrupt_t_0p6_0p8=0.9117 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9608 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8353 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=8500 epoch=8500/8500 epoch_step=1/1 micro_steps=8500 elapsed=6.6s lr=2.000000e-03 loss=0.8395 loss_recon=0.8395 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6039 corrupt_frac=0.9977 acc_corrupt=0.6030 loss_corrupt=0.8395 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5252 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5909 out_g_norm=0.3712 acc_corrupt_t_0p6_0p8=0.9132 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7714 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=8501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=8550 epoch=8550/9000 epoch_step=1/1 micro_steps=8550 elapsed=7.5s lr=2.000000e-03 loss=0.8415 loss_recon=0.8415 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6035 corrupt_frac=0.9977 acc_corrupt=0.6026 loss_corrupt=0.8415 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5246 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5940 out_g_norm=0.3549 acc_corrupt_t_0p8_1p0=0.9741 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8789 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=8600 epoch=8600/9000 epoch_step=1/1 micro_steps=8600 elapsed=6.5s lr=2.000000e-03 loss=0.8244 loss_recon=0.8244 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8244 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5274 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5994 out_g_norm=0.3632 acc_corrupt_t_0p8_1p0=0.9821 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8280 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=8650 epoch=8650/9000 epoch_step=1/1 micro_steps=8650 elapsed=6.5s lr=2.000000e-03 loss=0.8236 loss_recon=0.8236 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6070 corrupt_frac=0.9977 acc_corrupt=0.6061 loss_corrupt=0.8236 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5289 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.6069 out_g_norm=0.3725 loss_all=0.8210 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=8700 epoch=8700/9000 epoch_step=1/1 micro_steps=8700 elapsed=6.5s lr=2.000000e-03 loss=0.8323 loss_recon=0.8323 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6046 corrupt_frac=0.9977 acc_corrupt=0.6037 loss_corrupt=0.8323 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5247 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.6079 out_g_norm=0.4132 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7734 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=8750 epoch=8750/9000 epoch_step=1/1 micro_steps=8750 elapsed=6.5s lr=2.000000e-03 loss=0.8460 loss_recon=0.8460 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6039 corrupt_frac=0.9977 acc_corrupt=0.6030 loss_corrupt=0.8460 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5232 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8122 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.6024 out_g_norm=0.3873 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9706 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7890 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=8800 epoch=8800/9000 epoch_step=1/1 micro_steps=8800 elapsed=6.5s lr=2.000000e-03 loss=0.8303 loss_recon=0.8303 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6063 corrupt_frac=0.9977 acc_corrupt=0.6054 loss_corrupt=0.8303 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5272 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5959 out_g_norm=0.3486 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7766 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=8850 epoch=8850/9000 epoch_step=1/1 micro_steps=8850 elapsed=6.5s lr=2.000000e-03 loss=0.8480 loss_recon=0.8480 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6021 corrupt_frac=0.9977 acc_corrupt=0.6012 loss_corrupt=0.8480 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5226 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6741 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5937 out_g_norm=0.3239 loss_all=0.9434 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=8900 epoch=8900/9000 epoch_step=1/1 micro_steps=8900 elapsed=6.5s lr=2.000000e-03 loss=0.8285 loss_recon=0.8285 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6074 corrupt_frac=0.9977 acc_corrupt=0.6066 loss_corrupt=0.8285 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5268 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5858 out_g_norm=0.3513 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8122 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=8950 epoch=8950/9000 epoch_step=1/1 micro_steps=8950 elapsed=6.5s lr=2.000000e-03 loss=0.8429 loss_recon=0.8429 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6052 corrupt_frac=0.9977 acc_corrupt=0.6043 loss_corrupt=0.8429 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5237 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5835 out_g_norm=0.3343 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9755 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8328 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=9000 epoch=9000/9000 epoch_step=1/1 micro_steps=9000 elapsed=6.5s lr=2.000000e-03 loss=0.8494 loss_recon=0.8494 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6019 corrupt_frac=0.9977 acc_corrupt=0.6010 loss_corrupt=0.8494 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5224 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6737 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8101 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5746 out_g_norm=0.4026 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9510 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8070 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=9001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=9050 epoch=9050/9500 epoch_step=1/1 micro_steps=9050 elapsed=7.7s lr=2.000000e-03 loss=0.8321 loss_recon=0.8321 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6047 corrupt_frac=0.9977 acc_corrupt=0.6038 loss_corrupt=0.8321 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5264 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5799 out_g_norm=0.4013 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8447 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=9100 epoch=9100/9500 epoch_step=1/1 micro_steps=9100 elapsed=6.6s lr=2.000000e-03 loss=0.8239 loss_recon=0.8239 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8239 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5270 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9061 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5922 out_g_norm=0.3829 acc_corrupt_t_0p8_1p0=0.9817 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8777 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=9150 epoch=9150/9500 epoch_step=1/1 micro_steps=9150 elapsed=6.7s lr=2.000000e-03 loss=0.8269 loss_recon=0.8269 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6060 corrupt_frac=0.9977 acc_corrupt=0.6051 loss_corrupt=0.8269 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5273 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8097 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5907 out_g_norm=0.3596 loss_all=0.8034 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=9200 epoch=9200/9500 epoch_step=1/1 micro_steps=9200 elapsed=6.6s lr=2.000000e-03 loss=0.8354 loss_recon=0.8354 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6046 corrupt_frac=0.9977 acc_corrupt=0.6037 loss_corrupt=0.8354 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5245 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9085 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5830 out_g_norm=0.3359 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7930 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=9250 epoch=9250/9500 epoch_step=1/1 micro_steps=9250 elapsed=6.6s lr=2.000000e-03 loss=0.8367 loss_recon=0.8367 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6050 corrupt_frac=0.9977 acc_corrupt=0.6041 loss_corrupt=0.8367 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5248 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.5839 out_g_norm=0.3742 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8173 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=9300 epoch=9300/9500 epoch_step=1/1 micro_steps=9300 elapsed=6.6s lr=2.000000e-03 loss=0.8332 loss_recon=0.8332 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6058 corrupt_frac=0.9977 acc_corrupt=0.6049 loss_corrupt=0.8332 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5263 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5753 out_g_norm=0.3297 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7657 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=9350 epoch=9350/9500 epoch_step=1/1 micro_steps=9350 elapsed=6.6s lr=2.000000e-03 loss=0.8397 loss_recon=0.8397 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6036 corrupt_frac=0.9977 acc_corrupt=0.6027 loss_corrupt=0.8397 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5251 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6745 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8124 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5773 out_g_norm=0.3411 loss_all=0.8208 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=9400 epoch=9400/9500 epoch_step=1/1 micro_steps=9400 elapsed=6.6s lr=2.000000e-03 loss=0.8158 loss_recon=0.8158 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6098 corrupt_frac=0.9977 acc_corrupt=0.6090 loss_corrupt=0.8158 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5309 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5800 out_g_norm=0.3437 acc_corrupt_t_0p8_1p0=0.9677 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8067 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=9450 epoch=9450/9500 epoch_step=1/1 micro_steps=9450 elapsed=6.6s lr=2.000000e-03 loss=0.8282 loss_recon=0.8282 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6072 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8282 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5266 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5800 out_g_norm=0.3419 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7881 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=9500 epoch=9500/9500 epoch_step=1/1 micro_steps=9500 elapsed=6.6s lr=2.000000e-03 loss=0.8391 loss_recon=0.8391 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6035 corrupt_frac=0.9977 acc_corrupt=0.6026 loss_corrupt=0.8391 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5244 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8102 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5744 out_g_norm=0.4153 acc_corrupt_t_0p6_0p8=0.9135 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7636 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=9501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=9550 epoch=9550/10000 epoch_step=1/1 micro_steps=9550 elapsed=7.5s lr=2.000000e-03 loss=0.8320 loss_recon=0.8320 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6055 corrupt_frac=0.9977 acc_corrupt=0.6046 loss_corrupt=0.8320 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5282 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5746 out_g_norm=0.5169 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8147 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=9600 epoch=9600/10000 epoch_step=1/1 micro_steps=9600 elapsed=6.9s lr=2.000000e-03 loss=0.8159 loss_recon=0.8159 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6087 corrupt_frac=0.9977 acc_corrupt=0.6079 loss_corrupt=0.8159 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5299 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9058 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5812 out_g_norm=0.3256 acc_corrupt_t_0p8_1p0=0.9791 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7878 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=9650 epoch=9650/10000 epoch_step=1/1 micro_steps=9650 elapsed=6.6s lr=2.000000e-03 loss=0.8245 loss_recon=0.8245 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6066 corrupt_frac=0.9977 acc_corrupt=0.6057 loss_corrupt=0.8245 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5281 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8101 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5839 out_g_norm=0.4626 loss_all=0.8571 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=9700 epoch=9700/10000 epoch_step=1/1 micro_steps=9700 elapsed=6.7s lr=2.000000e-03 loss=0.8367 loss_recon=0.8367 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6048 corrupt_frac=0.9977 acc_corrupt=0.6039 loss_corrupt=0.8367 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5249 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8142 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5763 out_g_norm=0.4604 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7753 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=9750 epoch=9750/10000 epoch_step=1/1 micro_steps=9750 elapsed=6.8s lr=2.000000e-03 loss=0.8275 loss_recon=0.8275 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6067 corrupt_frac=0.9977 acc_corrupt=0.6058 loss_corrupt=0.8275 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5279 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6747 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.5666 out_g_norm=0.3284 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8374 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=9800 epoch=9800/10000 epoch_step=1/1 micro_steps=9800 elapsed=6.8s lr=2.000000e-03 loss=0.8320 loss_recon=0.8320 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6059 corrupt_frac=0.9977 acc_corrupt=0.6050 loss_corrupt=0.8320 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5266 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8104 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9087 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5583 out_g_norm=0.3648 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7690 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=9850 epoch=9850/10000 epoch_step=1/1 micro_steps=9850 elapsed=6.8s lr=2.000000e-03 loss=0.8441 loss_recon=0.8441 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6026 corrupt_frac=0.9977 acc_corrupt=0.6017 loss_corrupt=0.8441 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5231 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5619 out_g_norm=0.3572 loss_all=0.8802 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=9900 epoch=9900/10000 epoch_step=1/1 micro_steps=9900 elapsed=6.8s lr=2.000000e-03 loss=0.8113 loss_recon=0.8113 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.8113 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8128 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5647 out_g_norm=0.3412 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7707 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=9950 epoch=9950/10000 epoch_step=1/1 micro_steps=9950 elapsed=6.8s lr=2.000000e-03 loss=0.8321 loss_recon=0.8321 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6068 corrupt_frac=0.9977 acc_corrupt=0.6059 loss_corrupt=0.8321 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5262 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5615 out_g_norm=0.3799 acc_corrupt_t_0p6_0p8=0.9114 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7882 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=10000 epoch=10000/10000 epoch_step=1/1 micro_steps=10000 elapsed=6.8s lr=2.000000e-03 loss=0.8231 loss_recon=0.8231 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6065 corrupt_frac=0.9977 acc_corrupt=0.6057 loss_corrupt=0.8231 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5294 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5665 out_g_norm=0.3382 acc_corrupt_t_0p6_0p8=0.9130 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7519 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=10001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=10050 epoch=10050/10500 epoch_step=1/1 micro_steps=10050 elapsed=7.5s lr=2.000000e-03 loss=0.8249 loss_recon=0.8249 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6053 corrupt_frac=0.9977 acc_corrupt=0.6044 loss_corrupt=0.8249 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5275 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5666 out_g_norm=0.3963 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8102 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=10100 epoch=10100/10500 epoch_step=1/1 micro_steps=10100 elapsed=6.5s lr=2.000000e-03 loss=0.8247 loss_recon=0.8247 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6072 corrupt_frac=0.9977 acc_corrupt=0.6063 loss_corrupt=0.8247 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5275 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9036 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5648 out_g_norm=0.3371 acc_corrupt_t_0p8_1p0=0.9830 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7621 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=10150 epoch=10150/10500 epoch_step=1/1 micro_steps=10150 elapsed=6.6s lr=2.000000e-03 loss=0.8220 loss_recon=0.8220 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6064 corrupt_frac=0.9977 acc_corrupt=0.6055 loss_corrupt=0.8220 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5280 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6754 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8100 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5650 out_g_norm=0.4003 loss_all=0.8025 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=10200 epoch=10200/10500 epoch_step=1/1 micro_steps=10200 elapsed=6.7s lr=2.000000e-03 loss=0.8321 loss_recon=0.8321 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6051 corrupt_frac=0.9977 acc_corrupt=0.6043 loss_corrupt=0.8321 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5257 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6761 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9077 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5663 out_g_norm=0.3641 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8690 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=10250 epoch=10250/10500 epoch_step=1/1 micro_steps=10250 elapsed=6.6s lr=2.000000e-03 loss=0.8234 loss_recon=0.8234 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6063 corrupt_frac=0.9977 acc_corrupt=0.6055 loss_corrupt=0.8234 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5275 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6743 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.5598 out_g_norm=0.3356 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7825 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=10300 epoch=10300/10500 epoch_step=1/1 micro_steps=10300 elapsed=6.6s lr=2.000000e-03 loss=0.8288 loss_recon=0.8288 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6067 corrupt_frac=0.9977 acc_corrupt=0.6058 loss_corrupt=0.8288 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5282 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5556 out_g_norm=0.3286 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7487 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=10350 epoch=10350/10500 epoch_step=1/1 micro_steps=10350 elapsed=6.7s lr=2.000000e-03 loss=0.8297 loss_recon=0.8297 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6045 corrupt_frac=0.9977 acc_corrupt=0.6036 loss_corrupt=0.8297 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5261 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5589 out_g_norm=0.3255 loss_all=0.8271 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=10400 epoch=10400/10500 epoch_step=1/1 micro_steps=10400 elapsed=6.7s lr=2.000000e-03 loss=0.8178 loss_recon=0.8178 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8178 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5299 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5593 out_g_norm=0.3740 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8833 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=10450 epoch=10450/10500 epoch_step=1/1 micro_steps=10450 elapsed=6.7s lr=2.000000e-03 loss=0.8258 loss_recon=0.8258 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6078 corrupt_frac=0.9977 acc_corrupt=0.6070 loss_corrupt=0.8258 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5277 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5518 out_g_norm=0.3702 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9745 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8723 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=10500 epoch=10500/10500 epoch_step=1/1 micro_steps=10500 elapsed=6.7s lr=2.000000e-03 loss=0.8209 loss_recon=0.8209 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6062 corrupt_frac=0.9977 acc_corrupt=0.6053 loss_corrupt=0.8209 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5289 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5445 out_g_norm=0.4292 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9676 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8115 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=10501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=10550 epoch=10550/11000 epoch_step=1/1 micro_steps=10550 elapsed=7.4s lr=2.000000e-03 loss=0.8287 loss_recon=0.8287 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6063 corrupt_frac=0.9977 acc_corrupt=0.6054 loss_corrupt=0.8287 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5289 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5387 out_g_norm=0.3994 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8759 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=10600 epoch=10600/11000 epoch_step=1/1 micro_steps=10600 elapsed=6.5s lr=2.000000e-03 loss=0.8136 loss_recon=0.8136 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6085 corrupt_frac=0.9977 acc_corrupt=0.6076 loss_corrupt=0.8136 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5294 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9046 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5352 out_g_norm=0.3342 acc_corrupt_t_0p8_1p0=0.9749 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7916 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=10650 epoch=10650/11000 epoch_step=1/1 micro_steps=10650 elapsed=6.6s lr=2.000000e-03 loss=0.8195 loss_recon=0.8195 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6065 loss_corrupt=0.8195 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5290 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5421 out_g_norm=0.3367 loss_all=0.8297 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=10700 epoch=10700/11000 epoch_step=1/1 micro_steps=10700 elapsed=6.6s lr=2.000000e-03 loss=0.8128 loss_recon=0.8128 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6088 corrupt_frac=0.9977 acc_corrupt=0.6079 loss_corrupt=0.8128 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5313 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5373 out_g_norm=0.3666 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8476 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=10750 epoch=10750/11000 epoch_step=1/1 micro_steps=10750 elapsed=6.6s lr=2.000000e-03 loss=0.8184 loss_recon=0.8184 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6083 corrupt_frac=0.9977 acc_corrupt=0.6075 loss_corrupt=0.8184 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5304 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.5306 out_g_norm=0.3330 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9794 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7826 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=10800 epoch=10800/11000 epoch_step=1/1 micro_steps=10800 elapsed=6.6s lr=2.000000e-03 loss=0.8286 loss_recon=0.8286 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6064 corrupt_frac=0.9977 acc_corrupt=0.6055 loss_corrupt=0.8286 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5272 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9089 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5257 out_g_norm=0.3395 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9228 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=10850 epoch=10850/11000 epoch_step=1/1 micro_steps=10850 elapsed=6.6s lr=2.000000e-03 loss=0.8246 loss_recon=0.8246 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6054 corrupt_frac=0.9977 acc_corrupt=0.6045 loss_corrupt=0.8246 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5280 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6749 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8123 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5192 out_g_norm=0.3547 loss_all=0.7804 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=10900 epoch=10900/11000 epoch_step=1/1 micro_steps=10900 elapsed=6.6s lr=2.000000e-03 loss=0.8071 loss_recon=0.8071 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6097 loss_corrupt=0.8071 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5316 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5217 out_g_norm=0.3274 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8314 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=10950 epoch=10950/11000 epoch_step=1/1 micro_steps=10950 elapsed=6.6s lr=2.000000e-03 loss=0.8262 loss_recon=0.8262 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8262 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5265 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5183 out_g_norm=0.3310 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8934 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=11000 epoch=11000/11000 epoch_step=1/1 micro_steps=11000 elapsed=6.6s lr=2.000000e-03 loss=0.8191 loss_recon=0.8191 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6070 corrupt_frac=0.9977 acc_corrupt=0.6061 loss_corrupt=0.8191 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5305 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6750 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.5132 out_g_norm=0.3798 acc_corrupt_t_0p6_0p8=0.9128 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9676 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7598 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=11001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=11050 epoch=11050/11500 epoch_step=1/1 micro_steps=11050 elapsed=7.5s lr=2.000000e-03 loss=0.8184 loss_recon=0.8184 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6069 corrupt_frac=0.9977 acc_corrupt=0.6061 loss_corrupt=0.8184 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5302 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6767 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5135 out_g_norm=0.3566 acc_corrupt_t_0p8_1p0=0.9760 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7882 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=11100 epoch=11100/11500 epoch_step=1/1 micro_steps=11100 elapsed=6.6s lr=2.000000e-03 loss=0.8118 loss_recon=0.8118 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6089 corrupt_frac=0.9977 acc_corrupt=0.6081 loss_corrupt=0.8118 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5303 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9052 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.5123 out_g_norm=0.3364 acc_corrupt_t_0p8_1p0=0.9752 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8435 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=11150 epoch=11150/11500 epoch_step=1/1 micro_steps=11150 elapsed=6.6s lr=2.000000e-03 loss=0.8199 loss_recon=0.8199 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8199 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5289 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5103 out_g_norm=0.3146 loss_all=0.7954 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=11200 epoch=11200/11500 epoch_step=1/1 micro_steps=11200 elapsed=6.5s lr=2.000000e-03 loss=0.8198 loss_recon=0.8198 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6072 corrupt_frac=0.9977 acc_corrupt=0.6063 loss_corrupt=0.8198 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5282 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8150 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.5113 out_g_norm=0.3093 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7935 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=11250 epoch=11250/11500 epoch_step=1/1 micro_steps=11250 elapsed=6.5s lr=2.000000e-03 loss=0.8217 loss_recon=0.8217 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6075 corrupt_frac=0.9977 acc_corrupt=0.6066 loss_corrupt=0.8217 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5290 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6750 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4966 out_g_norm=0.3999 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8257 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=11300 epoch=11300/11500 epoch_step=1/1 micro_steps=11300 elapsed=6.5s lr=2.000000e-03 loss=0.8170 loss_recon=0.8170 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6079 corrupt_frac=0.9977 acc_corrupt=0.6070 loss_corrupt=0.8170 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5299 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4919 out_g_norm=0.3734 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7625 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=11350 epoch=11350/11500 epoch_step=1/1 micro_steps=11350 elapsed=6.5s lr=2.000000e-03 loss=0.8117 loss_recon=0.8117 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6075 corrupt_frac=0.9977 acc_corrupt=0.6066 loss_corrupt=0.8117 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5314 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4999 out_g_norm=0.3782 loss_all=0.7980 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=11400 epoch=11400/11500 epoch_step=1/1 micro_steps=11400 elapsed=6.5s lr=2.000000e-03 loss=0.8082 loss_recon=0.8082 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6102 corrupt_frac=0.9977 acc_corrupt=0.6093 loss_corrupt=0.8082 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5313 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.5018 out_g_norm=0.3601 acc_corrupt_t_0p8_1p0=0.9702 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7646 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=11450 epoch=11450/11500 epoch_step=1/1 micro_steps=11450 elapsed=6.5s lr=2.000000e-03 loss=0.8197 loss_recon=0.8197 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6085 corrupt_frac=0.9977 acc_corrupt=0.6077 loss_corrupt=0.8197 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5286 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.5003 out_g_norm=0.4017 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9740 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8375 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=11500 epoch=11500/11500 epoch_step=1/1 micro_steps=11500 elapsed=6.5s lr=2.000000e-03 loss=0.8184 loss_recon=0.8184 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6067 corrupt_frac=0.9977 acc_corrupt=0.6058 loss_corrupt=0.8184 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5293 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4963 out_g_norm=0.3211 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9686 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7875 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=11501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=11550 epoch=11550/12000 epoch_step=1/1 micro_steps=11550 elapsed=7.6s lr=2.000000e-03 loss=0.8138 loss_recon=0.8138 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6078 corrupt_frac=0.9977 acc_corrupt=0.6069 loss_corrupt=0.8138 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5314 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4962 out_g_norm=0.3483 acc_corrupt_t_0p8_1p0=0.9809 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7833 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=11600 epoch=11600/12000 epoch_step=1/1 micro_steps=11600 elapsed=6.6s lr=2.000000e-03 loss=0.8069 loss_recon=0.8069 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6096 loss_corrupt=0.8069 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8103 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9046 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4959 out_g_norm=0.3348 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8032 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=11650 epoch=11650/12000 epoch_step=1/1 micro_steps=11650 elapsed=6.5s lr=2.000000e-03 loss=0.8197 loss_recon=0.8197 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6074 corrupt_frac=0.9977 acc_corrupt=0.6066 loss_corrupt=0.8197 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5291 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.5005 out_g_norm=0.3775 loss_all=0.7909 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=11700 epoch=11700/12000 epoch_step=1/1 micro_steps=11700 elapsed=6.5s lr=2.000000e-03 loss=0.8124 loss_recon=0.8124 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6083 corrupt_frac=0.9977 acc_corrupt=0.6074 loss_corrupt=0.8124 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5301 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8148 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4979 out_g_norm=0.3356 acc_corrupt_t_0p8_1p0=0.9756 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7936 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=11750 epoch=11750/12000 epoch_step=1/1 micro_steps=11750 elapsed=6.5s lr=2.000000e-03 loss=0.8179 loss_recon=0.8179 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6077 corrupt_frac=0.9977 acc_corrupt=0.6068 loss_corrupt=0.8179 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5288 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4960 out_g_norm=0.3130 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8169 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=11800 epoch=11800/12000 epoch_step=1/1 micro_steps=11800 elapsed=6.6s lr=2.000000e-03 loss=0.8152 loss_recon=0.8152 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6080 corrupt_frac=0.9977 acc_corrupt=0.6071 loss_corrupt=0.8152 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5299 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5040 out_g_norm=0.3401 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7508 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=11850 epoch=11850/12000 epoch_step=1/1 micro_steps=11850 elapsed=6.6s lr=2.000000e-03 loss=0.8146 loss_recon=0.8146 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6070 corrupt_frac=0.9977 acc_corrupt=0.6061 loss_corrupt=0.8146 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.5071 out_g_norm=0.3679 loss_all=0.7923 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=11900 epoch=11900/12000 epoch_step=1/1 micro_steps=11900 elapsed=6.6s lr=2.000000e-03 loss=0.8119 loss_recon=0.8119 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8119 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5297 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4952 out_g_norm=0.3394 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8129 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=11950 epoch=11950/12000 epoch_step=1/1 micro_steps=11950 elapsed=6.7s lr=2.000000e-03 loss=0.8142 loss_recon=0.8142 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6084 loss_corrupt=0.8142 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5297 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4870 out_g_norm=0.3387 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9165 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=12000 epoch=12000/12000 epoch_step=1/1 micro_steps=12000 elapsed=6.6s lr=2.000000e-03 loss=0.8214 loss_recon=0.8214 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6060 corrupt_frac=0.9977 acc_corrupt=0.6051 loss_corrupt=0.8214 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5285 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4869 out_g_norm=0.3574 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9676 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8128 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=12001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=12050 epoch=12050/12500 epoch_step=1/1 micro_steps=12050 elapsed=7.7s lr=2.000000e-03 loss=0.8215 loss_recon=0.8215 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6063 corrupt_frac=0.9977 acc_corrupt=0.6055 loss_corrupt=0.8215 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5291 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4890 out_g_norm=0.3178 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8329 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=12100 epoch=12100/12500 epoch_step=1/1 micro_steps=12100 elapsed=6.6s lr=2.000000e-03 loss=0.8143 loss_recon=0.8143 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6093 corrupt_frac=0.9977 acc_corrupt=0.6084 loss_corrupt=0.8143 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5307 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9046 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4865 out_g_norm=0.3337 acc_corrupt_t_0p8_1p0=0.9798 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8164 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=12150 epoch=12150/12500 epoch_step=1/1 micro_steps=12150 elapsed=6.6s lr=2.000000e-03 loss=0.8128 loss_recon=0.8128 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6084 corrupt_frac=0.9977 acc_corrupt=0.6075 loss_corrupt=0.8128 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5308 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4866 out_g_norm=0.3056 loss_all=0.7672 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=12200 epoch=12200/12500 epoch_step=1/1 micro_steps=12200 elapsed=6.6s lr=2.000000e-03 loss=0.8144 loss_recon=0.8144 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6082 corrupt_frac=0.9977 acc_corrupt=0.6073 loss_corrupt=0.8144 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5305 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8145 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4781 out_g_norm=0.3066 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7710 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=12250 epoch=12250/12500 epoch_step=1/1 micro_steps=12250 elapsed=6.7s lr=2.000000e-03 loss=0.8074 loss_recon=0.8074 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6096 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8074 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5328 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6752 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4708 out_g_norm=0.3177 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7702 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=12300 epoch=12300/12500 epoch_step=1/1 micro_steps=12300 elapsed=6.7s lr=2.000000e-03 loss=0.8190 loss_recon=0.8190 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6074 corrupt_frac=0.9977 acc_corrupt=0.6065 loss_corrupt=0.8190 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5288 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4653 out_g_norm=0.3090 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7572 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=12350 epoch=12350/12500 epoch_step=1/1 micro_steps=12350 elapsed=6.7s lr=2.000000e-03 loss=0.8295 loss_recon=0.8295 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6044 corrupt_frac=0.9977 acc_corrupt=0.6035 loss_corrupt=0.8295 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5266 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6742 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8125 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4664 out_g_norm=0.3398 loss_all=0.8818 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=12400 epoch=12400/12500 epoch_step=1/1 micro_steps=12400 elapsed=6.5s lr=2.000000e-03 loss=0.7992 loss_recon=0.7992 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6114 corrupt_frac=0.9977 acc_corrupt=0.6105 loss_corrupt=0.7992 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5333 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4665 out_g_norm=0.3281 acc_corrupt_t_0p8_1p0=0.9697 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7802 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=12450 epoch=12450/12500 epoch_step=1/1 micro_steps=12450 elapsed=6.6s lr=2.000000e-03 loss=0.8127 loss_recon=0.8127 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6090 corrupt_frac=0.9977 acc_corrupt=0.6081 loss_corrupt=0.8127 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5296 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4727 out_g_norm=0.3435 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8249 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=12500 epoch=12500/12500 epoch_step=1/1 micro_steps=12500 elapsed=6.6s lr=2.000000e-03 loss=0.8199 loss_recon=0.8199 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6068 corrupt_frac=0.9977 acc_corrupt=0.6059 loss_corrupt=0.8199 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5296 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4712 out_g_norm=0.2986 acc_corrupt_t_0p6_0p8=0.9135 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9618 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7652 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=12501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=12550 epoch=12550/13000 epoch_step=1/1 micro_steps=12550 elapsed=7.5s lr=2.000000e-03 loss=0.8198 loss_recon=0.8198 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6059 corrupt_frac=0.9977 acc_corrupt=0.6051 loss_corrupt=0.8198 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5282 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9115 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4652 out_g_norm=0.2918 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8000 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=12600 epoch=12600/13000 epoch_step=1/1 micro_steps=12600 elapsed=6.6s lr=2.000000e-03 loss=0.8028 loss_recon=0.8028 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6107 corrupt_frac=0.9977 acc_corrupt=0.6098 loss_corrupt=0.8028 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5328 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9044 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4611 out_g_norm=0.3205 acc_corrupt_t_0p8_1p0=0.9788 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8332 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=12650 epoch=12650/13000 epoch_step=1/1 micro_steps=12650 elapsed=6.6s lr=2.000000e-03 loss=0.8103 loss_recon=0.8103 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6087 corrupt_frac=0.9977 acc_corrupt=0.6078 loss_corrupt=0.8103 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5317 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8105 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4618 out_g_norm=0.4614 loss_all=0.8613 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=12700 epoch=12700/13000 epoch_step=1/1 micro_steps=12700 elapsed=6.6s lr=2.000000e-03 loss=0.8214 loss_recon=0.8214 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6072 corrupt_frac=0.9977 acc_corrupt=0.6063 loss_corrupt=0.8214 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5284 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8146 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4536 out_g_norm=0.3790 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7753 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=12750 epoch=12750/13000 epoch_step=1/1 micro_steps=12750 elapsed=6.6s lr=2.000000e-03 loss=0.8088 loss_recon=0.8088 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6091 corrupt_frac=0.9977 acc_corrupt=0.6082 loss_corrupt=0.8088 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5314 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4506 out_g_norm=0.3311 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7968 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=12800 epoch=12800/13000 epoch_step=1/1 micro_steps=12800 elapsed=6.5s lr=2.000000e-03 loss=0.8207 loss_recon=0.8207 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6064 loss_corrupt=0.8207 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5285 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4580 out_g_norm=0.3304 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8093 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=12850 epoch=12850/13000 epoch_step=1/1 micro_steps=12850 elapsed=6.5s lr=2.000000e-03 loss=0.8138 loss_recon=0.8138 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6076 corrupt_frac=0.9977 acc_corrupt=0.6068 loss_corrupt=0.8138 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5317 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4592 out_g_norm=0.3082 loss_all=0.8284 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=12900 epoch=12900/13000 epoch_step=1/1 micro_steps=12900 elapsed=6.5s lr=2.000000e-03 loss=0.7991 loss_recon=0.7991 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6113 corrupt_frac=0.9977 acc_corrupt=0.6104 loss_corrupt=0.7991 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4649 out_g_norm=0.2987 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7614 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=12950 epoch=12950/13000 epoch_step=1/1 micro_steps=12950 elapsed=6.5s lr=2.000000e-03 loss=0.8114 loss_recon=0.8114 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8114 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5297 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4633 out_g_norm=0.3069 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9760 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9006 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=13000 epoch=13000/13000 epoch_step=1/1 micro_steps=13000 elapsed=6.5s lr=2.000000e-03 loss=0.8051 loss_recon=0.8051 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6090 corrupt_frac=0.9977 acc_corrupt=0.6082 loss_corrupt=0.8051 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5332 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4581 out_g_norm=0.3240 acc_corrupt_t_0p6_0p8=0.9120 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9725 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7518 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=13001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=13050 epoch=13050/13500 epoch_step=1/1 micro_steps=13050 elapsed=7.9s lr=2.000000e-03 loss=0.8094 loss_recon=0.8094 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6086 corrupt_frac=0.9977 acc_corrupt=0.6077 loss_corrupt=0.8094 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4531 out_g_norm=0.3628 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8166 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=13100 epoch=13100/13500 epoch_step=1/1 micro_steps=13100 elapsed=6.5s lr=2.000000e-03 loss=0.8020 loss_recon=0.8020 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6101 corrupt_frac=0.9977 acc_corrupt=0.6093 loss_corrupt=0.8020 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5320 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9038 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4535 out_g_norm=0.3747 acc_corrupt_t_0p8_1p0=0.9801 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7782 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=13150 epoch=13150/13500 epoch_step=1/1 micro_steps=13150 elapsed=6.5s lr=2.000000e-03 loss=0.8104 loss_recon=0.8104 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6087 corrupt_frac=0.9977 acc_corrupt=0.6078 loss_corrupt=0.8104 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5315 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4435 out_g_norm=0.3294 loss_all=0.8160 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=13200 epoch=13200/13500 epoch_step=1/1 micro_steps=13200 elapsed=6.5s lr=2.000000e-03 loss=0.8139 loss_recon=0.8139 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6077 corrupt_frac=0.9977 acc_corrupt=0.6068 loss_corrupt=0.8139 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5295 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4431 out_g_norm=0.3257 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8062 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=13250 epoch=13250/13500 epoch_step=1/1 micro_steps=13250 elapsed=6.5s lr=2.000000e-03 loss=0.8149 loss_recon=0.8149 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6086 corrupt_frac=0.9977 acc_corrupt=0.6077 loss_corrupt=0.8149 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6755 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4335 out_g_norm=0.3300 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8223 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=13300 epoch=13300/13500 epoch_step=1/1 micro_steps=13300 elapsed=6.5s lr=2.000000e-03 loss=0.8038 loss_recon=0.8038 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6100 corrupt_frac=0.9977 acc_corrupt=0.6091 loss_corrupt=0.8038 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5332 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4328 out_g_norm=0.3457 acc_corrupt_t_0p8_1p0=0.9804 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7357 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=13350 epoch=13350/13500 epoch_step=1/1 micro_steps=13350 elapsed=6.6s lr=2.000000e-03 loss=0.8231 loss_recon=0.8231 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6058 corrupt_frac=0.9977 acc_corrupt=0.6049 loss_corrupt=0.8231 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5283 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4384 out_g_norm=0.3732 loss_all=0.8373 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=13400 epoch=13400/13500 epoch_step=1/1 micro_steps=13400 elapsed=6.6s lr=2.000000e-03 loss=0.7989 loss_recon=0.7989 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6115 corrupt_frac=0.9977 acc_corrupt=0.6106 loss_corrupt=0.7989 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5331 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6767 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4430 out_g_norm=0.3454 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7727 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=13450 epoch=13450/13500 epoch_step=1/1 micro_steps=13450 elapsed=6.5s lr=2.000000e-03 loss=0.8107 loss_recon=0.8107 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6091 corrupt_frac=0.9977 acc_corrupt=0.6082 loss_corrupt=0.8107 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5293 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4550 out_g_norm=0.3031 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9598 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7923 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=13500 epoch=13500/13500 epoch_step=1/1 micro_steps=13500 elapsed=6.5s lr=2.000000e-03 loss=0.8166 loss_recon=0.8166 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6069 corrupt_frac=0.9977 acc_corrupt=0.6060 loss_corrupt=0.8166 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5298 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4423 out_g_norm=0.3040 acc_corrupt_t_0p6_0p8=0.9130 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7616 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=13501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=13550 epoch=13550/14000 epoch_step=1/1 micro_steps=13550 elapsed=7.8s lr=2.000000e-03 loss=0.8040 loss_recon=0.8040 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6086 corrupt_frac=0.9977 acc_corrupt=0.6077 loss_corrupt=0.8040 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5329 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4343 out_g_norm=0.3091 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7758 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=13600 epoch=13600/14000 epoch_step=1/1 micro_steps=13600 elapsed=6.9s lr=2.000000e-03 loss=0.8062 loss_recon=0.8062 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8062 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5308 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9047 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4432 out_g_norm=0.3491 acc_corrupt_t_0p8_1p0=0.9808 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8109 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=13650 epoch=13650/14000 epoch_step=1/1 micro_steps=13650 elapsed=6.8s lr=2.000000e-03 loss=0.8103 loss_recon=0.8103 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6082 corrupt_frac=0.9977 acc_corrupt=0.6074 loss_corrupt=0.8103 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5305 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4383 out_g_norm=0.3765 loss_all=0.8487 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=13700 epoch=13700/14000 epoch_step=1/1 micro_steps=13700 elapsed=6.6s lr=2.000000e-03 loss=0.8004 loss_recon=0.8004 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8004 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5327 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9081 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4420 out_g_norm=0.3333 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7684 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=13750 epoch=13750/14000 epoch_step=1/1 micro_steps=13750 elapsed=6.9s lr=2.000000e-03 loss=0.8101 loss_recon=0.8101 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6089 corrupt_frac=0.9977 acc_corrupt=0.6080 loss_corrupt=0.8101 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5312 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6756 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4429 out_g_norm=0.3247 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7695 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=13800 epoch=13800/14000 epoch_step=1/1 micro_steps=13800 elapsed=6.9s lr=2.000000e-03 loss=0.8034 loss_recon=0.8034 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6090 loss_corrupt=0.8034 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5329 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8119 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4408 out_g_norm=0.3114 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7507 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=13850 epoch=13850/14000 epoch_step=1/1 micro_steps=13850 elapsed=6.9s lr=2.000000e-03 loss=0.8196 loss_recon=0.8196 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6069 corrupt_frac=0.9977 acc_corrupt=0.6060 loss_corrupt=0.8196 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5303 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6753 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4344 out_g_norm=0.2999 loss_all=0.8429 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=13900 epoch=13900/14000 epoch_step=1/1 micro_steps=13900 elapsed=6.7s lr=2.000000e-03 loss=0.7919 loss_recon=0.7919 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6122 corrupt_frac=0.9977 acc_corrupt=0.6113 loss_corrupt=0.7919 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4381 out_g_norm=0.3232 acc_corrupt_t_0p8_1p0=0.9702 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7925 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=13950 epoch=13950/14000 epoch_step=1/1 micro_steps=13950 elapsed=6.9s lr=2.000000e-03 loss=0.8056 loss_recon=0.8056 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6101 corrupt_frac=0.9977 acc_corrupt=0.6092 loss_corrupt=0.8056 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5313 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4412 out_g_norm=0.3090 acc_corrupt_t_0p6_0p8=0.9131 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8557 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=14000 epoch=14000/14000 epoch_step=1/1 micro_steps=14000 elapsed=6.9s lr=2.000000e-03 loss=0.8256 loss_recon=0.8256 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6055 corrupt_frac=0.9977 acc_corrupt=0.6046 loss_corrupt=0.8256 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5273 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4369 out_g_norm=0.3901 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9657 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7547 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=14001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=14050 epoch=14050/14500 epoch_step=1/1 micro_steps=14050 elapsed=7.7s lr=2.000000e-03 loss=0.8017 loss_recon=0.8017 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6101 corrupt_frac=0.9977 acc_corrupt=0.6092 loss_corrupt=0.8017 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5351 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4336 out_g_norm=0.3201 acc_corrupt_t_0p8_1p0=0.9174 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8066 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=14100 epoch=14100/14500 epoch_step=1/1 micro_steps=14100 elapsed=6.8s lr=2.000000e-03 loss=0.7917 loss_recon=0.7917 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6119 corrupt_frac=0.9977 acc_corrupt=0.6110 loss_corrupt=0.7917 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9054 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4374 out_g_norm=0.2901 acc_corrupt_t_0p8_1p0=0.9781 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7641 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=14150 epoch=14150/14500 epoch_step=1/1 micro_steps=14150 elapsed=6.8s lr=2.000000e-03 loss=0.8074 loss_recon=0.8074 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6091 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8074 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5317 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4381 out_g_norm=0.3206 loss_all=0.8140 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=14200 epoch=14200/14500 epoch_step=1/1 micro_steps=14200 elapsed=6.8s lr=2.000000e-03 loss=0.8032 loss_recon=0.8032 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6091 loss_corrupt=0.8032 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5333 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4321 out_g_norm=0.2950 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7774 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=14250 epoch=14250/14500 epoch_step=1/1 micro_steps=14250 elapsed=6.7s lr=2.000000e-03 loss=0.8107 loss_recon=0.8107 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6091 corrupt_frac=0.9977 acc_corrupt=0.6082 loss_corrupt=0.8107 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5311 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4286 out_g_norm=0.2887 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9058 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=14300 epoch=14300/14500 epoch_step=1/1 micro_steps=14300 elapsed=6.7s lr=2.000000e-03 loss=0.8128 loss_recon=0.8128 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6081 corrupt_frac=0.9977 acc_corrupt=0.6073 loss_corrupt=0.8128 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5300 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4237 out_g_norm=0.2965 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7970 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=14350 epoch=14350/14500 epoch_step=1/1 micro_steps=14350 elapsed=6.7s lr=2.000000e-03 loss=0.8119 loss_recon=0.8119 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6066 corrupt_frac=0.9977 acc_corrupt=0.6058 loss_corrupt=0.8119 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5300 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6751 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4224 out_g_norm=0.2959 loss_all=0.7740 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=14400 epoch=14400/14500 epoch_step=1/1 micro_steps=14400 elapsed=6.8s lr=2.000000e-03 loss=0.7947 loss_recon=0.7947 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6123 corrupt_frac=0.9977 acc_corrupt=0.6115 loss_corrupt=0.7947 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5351 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4187 out_g_norm=0.4296 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7882 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=14450 epoch=14450/14500 epoch_step=1/1 micro_steps=14450 elapsed=6.8s lr=2.000000e-03 loss=0.8176 loss_recon=0.8176 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6079 corrupt_frac=0.9977 acc_corrupt=0.6070 loss_corrupt=0.8176 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5275 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4092 out_g_norm=0.3267 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9740 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8998 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=14500 epoch=14500/14500 epoch_step=1/1 micro_steps=14500 elapsed=6.8s lr=2.000000e-03 loss=0.7972 loss_recon=0.7972 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6103 corrupt_frac=0.9977 acc_corrupt=0.6094 loss_corrupt=0.7972 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5353 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4020 out_g_norm=0.4020 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7402 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=14501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=14550 epoch=14550/15000 epoch_step=1/1 micro_steps=14550 elapsed=7.5s lr=2.000000e-03 loss=0.8005 loss_recon=0.8005 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6096 corrupt_frac=0.9977 acc_corrupt=0.6087 loss_corrupt=0.8005 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5341 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6779 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4153 out_g_norm=0.3763 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8072 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=14600 epoch=14600/15000 epoch_step=1/1 micro_steps=14600 elapsed=6.6s lr=2.000000e-03 loss=0.8081 loss_recon=0.8081 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8081 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9053 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4193 out_g_norm=0.3766 acc_corrupt_t_0p8_1p0=0.9817 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7888 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=14650 epoch=14650/15000 epoch_step=1/1 micro_steps=14650 elapsed=6.6s lr=2.000000e-03 loss=0.8110 loss_recon=0.8110 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6079 corrupt_frac=0.9977 acc_corrupt=0.6071 loss_corrupt=0.8110 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5300 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4209 out_g_norm=0.3359 loss_all=0.8045 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=14700 epoch=14700/15000 epoch_step=1/1 micro_steps=14700 elapsed=6.6s lr=2.000000e-03 loss=0.8120 loss_recon=0.8120 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6079 corrupt_frac=0.9977 acc_corrupt=0.6070 loss_corrupt=0.8120 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5294 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8152 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9091 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4153 out_g_norm=0.3161 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7705 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=14750 epoch=14750/15000 epoch_step=1/1 micro_steps=14750 elapsed=6.6s lr=2.000000e-03 loss=0.8055 loss_recon=0.8055 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6104 corrupt_frac=0.9977 acc_corrupt=0.6095 loss_corrupt=0.8055 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5333 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4124 out_g_norm=0.2977 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7746 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=14800 epoch=14800/15000 epoch_step=1/1 micro_steps=14800 elapsed=6.6s lr=2.000000e-03 loss=0.8159 loss_recon=0.8159 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6087 corrupt_frac=0.9977 acc_corrupt=0.6078 loss_corrupt=0.8159 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4202 out_g_norm=0.3256 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7674 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=14850 epoch=14850/15000 epoch_step=1/1 micro_steps=14850 elapsed=6.6s lr=2.000000e-03 loss=0.8157 loss_recon=0.8157 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6066 corrupt_frac=0.9977 acc_corrupt=0.6057 loss_corrupt=0.8157 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5301 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6748 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4150 out_g_norm=0.3075 loss_all=0.7856 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=14900 epoch=14900/15000 epoch_step=1/1 micro_steps=14900 elapsed=6.6s lr=2.000000e-03 loss=0.7896 loss_recon=0.7896 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6131 corrupt_frac=0.9977 acc_corrupt=0.6122 loss_corrupt=0.7896 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5358 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9106 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4207 out_g_norm=0.2963 acc_corrupt_t_0p8_1p0=0.9711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7803 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=14950 epoch=14950/15000 epoch_step=1/1 micro_steps=14950 elapsed=6.5s lr=2.000000e-03 loss=0.8042 loss_recon=0.8042 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6096 loss_corrupt=0.8042 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5316 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4254 out_g_norm=0.3628 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9740 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8955 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=15000 epoch=15000/15000 epoch_step=1/1 micro_steps=15000 elapsed=6.7s lr=2.000000e-03 loss=0.8090 loss_recon=0.8090 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6089 corrupt_frac=0.9977 acc_corrupt=0.6080 loss_corrupt=0.8090 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5326 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4214 out_g_norm=0.3506 acc_corrupt_t_0p6_0p8=0.9129 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9637 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7807 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=15001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=15050 epoch=15050/15500 epoch_step=1/1 micro_steps=15050 elapsed=7.5s lr=2.000000e-03 loss=0.7985 loss_recon=0.7985 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6096 corrupt_frac=0.9977 acc_corrupt=0.6087 loss_corrupt=0.7985 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5343 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4163 out_g_norm=0.2966 acc_corrupt_t_0p8_1p0=0.9809 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8386 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=15100 epoch=15100/15500 epoch_step=1/1 micro_steps=15100 elapsed=6.6s lr=2.000000e-03 loss=0.7929 loss_recon=0.7929 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6117 corrupt_frac=0.9977 acc_corrupt=0.6108 loss_corrupt=0.7929 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9047 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4121 out_g_norm=0.3013 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8082 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=15150 epoch=15150/15500 epoch_step=1/1 micro_steps=15150 elapsed=6.6s lr=2.000000e-03 loss=0.7985 loss_recon=0.7985 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6096 loss_corrupt=0.7985 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4087 out_g_norm=0.2911 loss_all=0.8305 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=15200 epoch=15200/15500 epoch_step=1/1 micro_steps=15200 elapsed=6.7s lr=2.000000e-03 loss=0.8085 loss_recon=0.8085 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6083 corrupt_frac=0.9977 acc_corrupt=0.6074 loss_corrupt=0.8085 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5300 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8149 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4046 out_g_norm=0.2974 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7674 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=15250 epoch=15250/15500 epoch_step=1/1 micro_steps=15250 elapsed=6.7s lr=2.000000e-03 loss=0.8027 loss_recon=0.8027 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6098 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.8027 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5325 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3997 out_g_norm=0.3192 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7991 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=15300 epoch=15300/15500 epoch_step=1/1 micro_steps=15300 elapsed=6.7s lr=2.000000e-03 loss=0.8190 loss_recon=0.8190 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6077 corrupt_frac=0.9977 acc_corrupt=0.6069 loss_corrupt=0.8190 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5294 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6767 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3984 out_g_norm=0.3615 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7921 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=15350 epoch=15350/15500 epoch_step=1/1 micro_steps=15350 elapsed=6.7s lr=2.000000e-03 loss=0.8102 loss_recon=0.8102 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6074 corrupt_frac=0.9977 acc_corrupt=0.6065 loss_corrupt=0.8102 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5315 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6751 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8126 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3935 out_g_norm=0.3271 loss_all=0.8408 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=15400 epoch=15400/15500 epoch_step=1/1 micro_steps=15400 elapsed=6.7s lr=2.000000e-03 loss=0.7917 loss_recon=0.7917 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6127 corrupt_frac=0.9977 acc_corrupt=0.6119 loss_corrupt=0.7917 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5354 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4043 out_g_norm=0.3261 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7596 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=15450 epoch=15450/15500 epoch_step=1/1 micro_steps=15450 elapsed=6.6s lr=2.000000e-03 loss=0.7998 loss_recon=0.7998 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6115 corrupt_frac=0.9977 acc_corrupt=0.6106 loss_corrupt=0.7998 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5335 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3986 out_g_norm=0.3163 acc_corrupt_t_0p6_0p8=0.9133 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8829 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=15500 epoch=15500/15500 epoch_step=1/1 micro_steps=15500 elapsed=6.5s lr=2.000000e-03 loss=0.8025 loss_recon=0.8025 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6098 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.8025 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5347 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3960 out_g_norm=0.3740 acc_corrupt_t_0p6_0p8=0.9126 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9696 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8261 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=15501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=15550 epoch=15550/16000 epoch_step=1/1 micro_steps=15550 elapsed=7.6s lr=2.000000e-03 loss=0.8024 loss_recon=0.8024 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6090 corrupt_frac=0.9977 acc_corrupt=0.6081 loss_corrupt=0.8024 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5336 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9079 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3931 out_g_norm=0.3088 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7790 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=15600 epoch=15600/16000 epoch_step=1/1 micro_steps=15600 elapsed=6.6s lr=2.000000e-03 loss=0.7928 loss_recon=0.7928 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6124 corrupt_frac=0.9977 acc_corrupt=0.6116 loss_corrupt=0.7928 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5356 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9054 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4007 out_g_norm=0.3308 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7947 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=15650 epoch=15650/16000 epoch_step=1/1 micro_steps=15650 elapsed=6.8s lr=2.000000e-03 loss=0.8031 loss_recon=0.8031 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6093 corrupt_frac=0.9977 acc_corrupt=0.6084 loss_corrupt=0.8031 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5323 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.4078 out_g_norm=0.3889 loss_all=0.7808 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=15700 epoch=15700/16000 epoch_step=1/1 micro_steps=15700 elapsed=6.7s lr=2.000000e-03 loss=0.8046 loss_recon=0.8046 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8046 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5315 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6780 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.4155 out_g_norm=0.3072 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8143 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=15750 epoch=15750/16000 epoch_step=1/1 micro_steps=15750 elapsed=6.7s lr=2.000000e-03 loss=0.7986 loss_recon=0.7986 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6110 corrupt_frac=0.9977 acc_corrupt=0.6102 loss_corrupt=0.7986 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5347 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.4112 out_g_norm=0.2918 acc_corrupt_t_0p6_0p8=0.9089 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9882 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7783 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=15800 epoch=15800/16000 epoch_step=1/1 micro_steps=15800 elapsed=6.8s lr=2.000000e-03 loss=0.8016 loss_recon=0.8016 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8016 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5324 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4177 out_g_norm=0.3171 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7845 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=15850 epoch=15850/16000 epoch_step=1/1 micro_steps=15850 elapsed=6.7s lr=2.000000e-03 loss=0.8057 loss_recon=0.8057 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6080 corrupt_frac=0.9977 acc_corrupt=0.6072 loss_corrupt=0.8057 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5320 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4170 out_g_norm=0.2883 loss_all=0.7873 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=15900 epoch=15900/16000 epoch_step=1/1 micro_steps=15900 elapsed=6.7s lr=2.000000e-03 loss=0.7943 loss_recon=0.7943 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6120 corrupt_frac=0.9977 acc_corrupt=0.6111 loss_corrupt=0.7943 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5337 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4134 out_g_norm=0.3555 acc_corrupt_t_0p8_1p0=0.9741 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7816 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=15950 epoch=15950/16000 epoch_step=1/1 micro_steps=15950 elapsed=6.7s lr=2.000000e-03 loss=0.7943 loss_recon=0.7943 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6119 corrupt_frac=0.9977 acc_corrupt=0.6111 loss_corrupt=0.7943 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5338 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4115 out_g_norm=0.3057 acc_corrupt_t_0p6_0p8=0.9132 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9740 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9027 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=16000 epoch=16000/16000 epoch_step=1/1 micro_steps=16000 elapsed=6.8s lr=2.000000e-03 loss=0.8045 loss_recon=0.8045 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=0.9977 acc_corrupt=0.6083 loss_corrupt=0.8045 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5332 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.4089 out_g_norm=0.3799 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9667 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7456 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=16001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=16050 epoch=16050/16500 epoch_step=1/1 micro_steps=16050 elapsed=7.7s lr=2.000000e-03 loss=0.7962 loss_recon=0.7962 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6090 loss_corrupt=0.7962 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5349 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4114 out_g_norm=0.3010 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8292 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=16100 epoch=16100/16500 epoch_step=1/1 micro_steps=16100 elapsed=6.5s lr=2.000000e-03 loss=0.7939 loss_recon=0.7939 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6116 corrupt_frac=0.9977 acc_corrupt=0.6108 loss_corrupt=0.7939 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5341 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6775 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9029 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4090 out_g_norm=0.3527 acc_corrupt_t_0p8_1p0=0.9791 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7710 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=16150 epoch=16150/16500 epoch_step=1/1 micro_steps=16150 elapsed=6.8s lr=2.000000e-03 loss=0.8069 loss_recon=0.8069 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6088 corrupt_frac=0.9977 acc_corrupt=0.6079 loss_corrupt=0.8069 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5314 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8106 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3982 out_g_norm=0.3079 loss_all=0.8102 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=16200 epoch=16200/16500 epoch_step=1/1 micro_steps=16200 elapsed=6.6s lr=2.000000e-03 loss=0.8035 loss_recon=0.8035 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6088 loss_corrupt=0.8035 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5326 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3958 out_g_norm=0.3078 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7544 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=16250 epoch=16250/16500 epoch_step=1/1 micro_steps=16250 elapsed=6.5s lr=2.000000e-03 loss=0.7968 loss_recon=0.7968 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6114 corrupt_frac=0.9977 acc_corrupt=0.6105 loss_corrupt=0.7968 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5349 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3972 out_g_norm=0.3321 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9882 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8050 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=16300 epoch=16300/16500 epoch_step=1/1 micro_steps=16300 elapsed=6.5s lr=2.000000e-03 loss=0.7994 loss_recon=0.7994 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6110 corrupt_frac=0.9977 acc_corrupt=0.6101 loss_corrupt=0.7994 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4059 out_g_norm=0.3925 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7672 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=16350 epoch=16350/16500 epoch_step=1/1 micro_steps=16350 elapsed=6.5s lr=2.000000e-03 loss=0.8148 loss_recon=0.8148 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6073 corrupt_frac=0.9977 acc_corrupt=0.6065 loss_corrupt=0.8148 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5308 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9104 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.4014 out_g_norm=0.3167 loss_all=0.7851 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=16400 epoch=16400/16500 epoch_step=1/1 micro_steps=16400 elapsed=6.7s lr=2.000000e-03 loss=0.7926 loss_recon=0.7926 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6127 corrupt_frac=0.9977 acc_corrupt=0.6118 loss_corrupt=0.7926 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5352 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9115 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4044 out_g_norm=0.3197 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8155 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=16450 epoch=16450/16500 epoch_step=1/1 micro_steps=16450 elapsed=7.0s lr=2.000000e-03 loss=0.8014 loss_recon=0.8014 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6106 corrupt_frac=0.9977 acc_corrupt=0.6098 loss_corrupt=0.8014 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5324 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3972 out_g_norm=0.3200 acc_corrupt_t_0p6_0p8=0.9126 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8598 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=16500 epoch=16500/16500 epoch_step=1/1 micro_steps=16500 elapsed=7.0s lr=2.000000e-03 loss=0.7958 loss_recon=0.7958 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6106 corrupt_frac=0.9977 acc_corrupt=0.6097 loss_corrupt=0.7958 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5356 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3954 out_g_norm=0.2831 acc_corrupt_t_0p6_0p8=0.9131 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9676 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8181 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=16501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=16550 epoch=16550/17000 epoch_step=1/1 micro_steps=16550 elapsed=7.5s lr=2.000000e-03 loss=0.7985 loss_recon=0.7985 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6104 corrupt_frac=0.9977 acc_corrupt=0.6095 loss_corrupt=0.7985 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5354 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6781 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9116 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3975 out_g_norm=0.2821 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8032 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=16600 epoch=16600/17000 epoch_step=1/1 micro_steps=16600 elapsed=6.5s lr=2.000000e-03 loss=0.7961 loss_recon=0.7961 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6112 corrupt_frac=0.9977 acc_corrupt=0.6104 loss_corrupt=0.7961 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5338 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.4017 out_g_norm=0.3239 acc_corrupt_t_0p8_1p0=0.9808 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8227 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=16650 epoch=16650/17000 epoch_step=1/1 micro_steps=16650 elapsed=6.6s lr=2.000000e-03 loss=0.7913 loss_recon=0.7913 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6113 corrupt_frac=0.9977 acc_corrupt=0.6105 loss_corrupt=0.7913 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5358 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3944 out_g_norm=0.3049 loss_all=0.8004 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=16700 epoch=16700/17000 epoch_step=1/1 micro_steps=16700 elapsed=6.6s lr=2.000000e-03 loss=0.8009 loss_recon=0.8009 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6103 corrupt_frac=0.9977 acc_corrupt=0.6094 loss_corrupt=0.8009 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5333 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6781 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8152 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3890 out_g_norm=0.2966 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7756 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=16750 epoch=16750/17000 epoch_step=1/1 micro_steps=16750 elapsed=6.5s lr=2.000000e-03 loss=0.7958 loss_recon=0.7958 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6108 corrupt_frac=0.9977 acc_corrupt=0.6100 loss_corrupt=0.7958 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5341 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3878 out_g_norm=0.3065 acc_corrupt_t_0p6_0p8=0.9103 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8080 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=16800 epoch=16800/17000 epoch_step=1/1 micro_steps=16800 elapsed=6.6s lr=2.000000e-03 loss=0.7969 loss_recon=0.7969 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6113 corrupt_frac=0.9977 acc_corrupt=0.6104 loss_corrupt=0.7969 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5348 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6782 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3906 out_g_norm=0.2818 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7998 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=16850 epoch=16850/17000 epoch_step=1/1 micro_steps=16850 elapsed=7.0s lr=2.000000e-03 loss=0.8000 loss_recon=0.8000 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6090 corrupt_frac=0.9977 acc_corrupt=0.6081 loss_corrupt=0.8000 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5337 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3956 out_g_norm=0.3126 loss_all=0.7830 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=16900 epoch=16900/17000 epoch_step=1/1 micro_steps=16900 elapsed=6.7s lr=2.000000e-03 loss=0.7972 loss_recon=0.7972 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6123 corrupt_frac=0.9977 acc_corrupt=0.6114 loss_corrupt=0.7972 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8139 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3950 out_g_norm=0.3866 acc_corrupt_t_0p8_1p0=0.9741 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7633 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=16950 epoch=16950/17000 epoch_step=1/1 micro_steps=16950 elapsed=6.5s lr=2.000000e-03 loss=0.7936 loss_recon=0.7936 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6124 corrupt_frac=0.9977 acc_corrupt=0.6115 loss_corrupt=0.7936 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5350 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3880 out_g_norm=0.3176 acc_corrupt_t_0p6_0p8=0.9133 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8270 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=17000 epoch=17000/17000 epoch_step=1/1 micro_steps=17000 elapsed=6.5s lr=2.000000e-03 loss=0.8095 loss_recon=0.8095 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6082 corrupt_frac=0.9977 acc_corrupt=0.6073 loss_corrupt=0.8095 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5314 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3978 out_g_norm=0.3044 acc_corrupt_t_0p6_0p8=0.9122 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9745 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7542 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=17001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=17050 epoch=17050/17500 epoch_step=1/1 micro_steps=17050 elapsed=7.4s lr=2.000000e-03 loss=0.8019 loss_recon=0.8019 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6094 corrupt_frac=0.9977 acc_corrupt=0.6086 loss_corrupt=0.8019 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5340 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3973 out_g_norm=0.3066 acc_corrupt_t_0p8_1p0=0.9614 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7898 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=17100 epoch=17100/17500 epoch_step=1/1 micro_steps=17100 elapsed=6.5s lr=2.000000e-03 loss=0.7917 loss_recon=0.7917 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6120 corrupt_frac=0.9977 acc_corrupt=0.6112 loss_corrupt=0.7917 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5350 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6774 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8108 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3981 out_g_norm=0.3154 acc_corrupt_t_0p8_1p0=0.9804 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8125 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=17150 epoch=17150/17500 epoch_step=1/1 micro_steps=17150 elapsed=6.5s lr=2.000000e-03 loss=0.7996 loss_recon=0.7996 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6106 corrupt_frac=0.9977 acc_corrupt=0.6097 loss_corrupt=0.7996 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5341 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9112 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3887 out_g_norm=0.2976 loss_all=0.8278 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=17200 epoch=17200/17500 epoch_step=1/1 micro_steps=17200 elapsed=6.5s lr=2.000000e-03 loss=0.7930 loss_recon=0.7930 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6112 corrupt_frac=0.9977 acc_corrupt=0.6103 loss_corrupt=0.7930 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5348 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6782 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8154 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3952 out_g_norm=0.3394 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7571 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=17250 epoch=17250/17500 epoch_step=1/1 micro_steps=17250 elapsed=6.5s lr=2.000000e-03 loss=0.7912 loss_recon=0.7912 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6124 corrupt_frac=0.9977 acc_corrupt=0.6116 loss_corrupt=0.7912 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5370 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6764 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3801 out_g_norm=0.3056 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7770 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=17300 epoch=17300/17500 epoch_step=1/1 micro_steps=17300 elapsed=6.5s lr=2.000000e-03 loss=0.7915 loss_recon=0.7915 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6114 corrupt_frac=0.9977 acc_corrupt=0.6106 loss_corrupt=0.7915 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5350 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6782 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8120 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9110 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3830 out_g_norm=0.3245 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7289 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=17350 epoch=17350/17500 epoch_step=1/1 micro_steps=17350 elapsed=6.5s lr=2.000000e-03 loss=0.8009 loss_recon=0.8009 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6093 corrupt_frac=0.9977 acc_corrupt=0.6084 loss_corrupt=0.8009 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6760 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9099 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3808 out_g_norm=0.2725 loss_all=0.8351 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=17400 epoch=17400/17500 epoch_step=1/1 micro_steps=17400 elapsed=6.5s lr=2.000000e-03 loss=0.7981 loss_recon=0.7981 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6121 corrupt_frac=0.9977 acc_corrupt=0.6112 loss_corrupt=0.7981 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5339 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9124 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3774 out_g_norm=0.3022 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7808 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=17450 epoch=17450/17500 epoch_step=1/1 micro_steps=17450 elapsed=6.5s lr=2.000000e-03 loss=0.7988 loss_recon=0.7988 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6108 corrupt_frac=0.9977 acc_corrupt=0.6099 loss_corrupt=0.7988 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5322 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8134 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3756 out_g_norm=0.2969 acc_corrupt_t_0p6_0p8=0.9135 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8320 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=17500 epoch=17500/17500 epoch_step=1/1 micro_steps=17500 elapsed=6.5s lr=2.000000e-03 loss=0.7982 loss_recon=0.7982 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6091 loss_corrupt=0.7982 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8113 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3713 out_g_norm=0.3112 acc_corrupt_t_0p6_0p8=0.9119 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9725 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7588 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=17501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=17550 epoch=17550/18000 epoch_step=1/1 micro_steps=17550 elapsed=7.6s lr=2.000000e-03 loss=0.7965 loss_recon=0.7965 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6101 corrupt_frac=0.9977 acc_corrupt=0.6092 loss_corrupt=0.7965 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5352 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9097 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3749 out_g_norm=0.3023 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7909 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=17600 epoch=17600/18000 epoch_step=1/1 micro_steps=17600 elapsed=6.7s lr=2.000000e-03 loss=0.7904 loss_recon=0.7904 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6122 corrupt_frac=0.9977 acc_corrupt=0.6114 loss_corrupt=0.7904 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5355 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9043 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3688 out_g_norm=0.3553 acc_corrupt_t_0p8_1p0=0.9804 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7870 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=17650 epoch=17650/18000 epoch_step=1/1 micro_steps=17650 elapsed=6.7s lr=2.000000e-03 loss=0.8059 loss_recon=0.8059 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6088 corrupt_frac=0.9977 acc_corrupt=0.6079 loss_corrupt=0.8059 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5315 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6765 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9090 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3553 out_g_norm=0.2803 loss_all=0.7857 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=17700 epoch=17700/18000 epoch_step=1/1 micro_steps=17700 elapsed=6.7s lr=2.000000e-03 loss=0.7982 loss_recon=0.7982 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6107 corrupt_frac=0.9977 acc_corrupt=0.6098 loss_corrupt=0.7982 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5339 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6783 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8153 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9092 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3594 out_g_norm=0.3083 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7582 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=17750 epoch=17750/18000 epoch_step=1/1 micro_steps=17750 elapsed=6.7s lr=2.000000e-03 loss=0.7968 loss_recon=0.7968 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6110 corrupt_frac=0.9977 acc_corrupt=0.6101 loss_corrupt=0.7968 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3735 out_g_norm=0.3107 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7739 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=17800 epoch=17800/18000 epoch_step=1/1 micro_steps=17800 elapsed=6.7s lr=2.000000e-03 loss=0.7967 loss_recon=0.7967 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6109 corrupt_frac=0.9977 acc_corrupt=0.6101 loss_corrupt=0.7967 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8121 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3713 out_g_norm=0.2981 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7668 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=17850 epoch=17850/18000 epoch_step=1/1 micro_steps=17850 elapsed=6.7s lr=2.000000e-03 loss=0.8032 loss_recon=0.8032 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6082 corrupt_frac=0.9977 acc_corrupt=0.6073 loss_corrupt=0.8032 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5321 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6763 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3655 out_g_norm=0.2823 loss_all=0.7909 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=17900 epoch=17900/18000 epoch_step=1/1 micro_steps=17900 elapsed=6.7s lr=2.000000e-03 loss=0.7919 loss_recon=0.7919 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6126 corrupt_frac=0.9977 acc_corrupt=0.6117 loss_corrupt=0.7919 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5349 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3588 out_g_norm=0.3105 acc_corrupt_t_0p8_1p0=0.9751 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7599 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=17950 epoch=17950/18000 epoch_step=1/1 micro_steps=17950 elapsed=6.7s lr=2.000000e-03 loss=0.7950 loss_recon=0.7950 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6122 corrupt_frac=0.9977 acc_corrupt=0.6114 loss_corrupt=0.7950 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3522 out_g_norm=0.3107 acc_corrupt_t_0p6_0p8=0.9124 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9745 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8557 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=18000 epoch=18000/18000 epoch_step=1/1 micro_steps=18000 elapsed=6.7s lr=2.000000e-03 loss=0.7958 loss_recon=0.7958 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6090 loss_corrupt=0.7958 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3588 out_g_norm=0.3091 acc_corrupt_t_0p6_0p8=0.9138 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9725 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7782 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=18001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=18050 epoch=18050/18500 epoch_step=1/1 micro_steps=18050 elapsed=7.4s lr=2.000000e-03 loss=0.7982 loss_recon=0.7982 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6098 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.7982 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6779 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3657 out_g_norm=0.3009 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7892 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=18100 epoch=18100/18500 epoch_step=1/1 micro_steps=18100 elapsed=6.5s lr=2.000000e-03 loss=0.7850 loss_recon=0.7850 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6133 corrupt_frac=0.9977 acc_corrupt=0.6125 loss_corrupt=0.7850 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5370 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8114 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9045 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3614 out_g_norm=0.2994 acc_corrupt_t_0p8_1p0=0.9798 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8127 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=18150 epoch=18150/18500 epoch_step=1/1 micro_steps=18150 elapsed=6.5s lr=2.000000e-03 loss=0.7990 loss_recon=0.7990 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6096 loss_corrupt=0.7990 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5343 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9105 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3559 out_g_norm=0.2805 loss_all=0.8164 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=18200 epoch=18200/18500 epoch_step=1/1 micro_steps=18200 elapsed=6.5s lr=2.000000e-03 loss=0.7893 loss_recon=0.7893 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6116 corrupt_frac=0.9977 acc_corrupt=0.6107 loss_corrupt=0.7893 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5360 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8151 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3491 out_g_norm=0.2787 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8072 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=18250 epoch=18250/18500 epoch_step=1/1 micro_steps=18250 elapsed=6.5s lr=2.000000e-03 loss=0.7954 loss_recon=0.7954 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6116 corrupt_frac=0.9977 acc_corrupt=0.6107 loss_corrupt=0.7954 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5353 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6766 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8129 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3402 out_g_norm=0.2797 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7852 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=18300 epoch=18300/18500 epoch_step=1/1 micro_steps=18300 elapsed=6.5s lr=2.000000e-03 loss=0.7959 loss_recon=0.7959 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6112 corrupt_frac=0.9977 acc_corrupt=0.6103 loss_corrupt=0.7959 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5348 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6780 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8118 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9100 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3315 out_g_norm=0.3350 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7285 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=18350 epoch=18350/18500 epoch_step=1/1 micro_steps=18350 elapsed=6.5s lr=2.000000e-03 loss=0.7995 loss_recon=0.7995 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6088 corrupt_frac=0.9977 acc_corrupt=0.6080 loss_corrupt=0.7995 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5335 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6758 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8135 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3237 out_g_norm=0.3407 loss_all=0.7844 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=18400 epoch=18400/18500 epoch_step=1/1 micro_steps=18400 elapsed=6.6s lr=2.000000e-03 loss=0.7910 loss_recon=0.7910 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6126 corrupt_frac=0.9977 acc_corrupt=0.6117 loss_corrupt=0.7910 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5347 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9111 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3255 out_g_norm=0.2874 acc_corrupt_t_0p8_1p0=0.9687 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7582 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=18450 epoch=18450/18500 epoch_step=1/1 micro_steps=18450 elapsed=6.6s lr=2.000000e-03 loss=0.7987 loss_recon=0.7987 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6116 corrupt_frac=0.9977 acc_corrupt=0.6107 loss_corrupt=0.7987 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5336 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3258 out_g_norm=0.2984 acc_corrupt_t_0p6_0p8=0.9128 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9726 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8454 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=18500 epoch=18500/18500 epoch_step=1/1 micro_steps=18500 elapsed=6.7s lr=2.000000e-03 loss=0.8009 loss_recon=0.8009 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.8009 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6767 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3252 out_g_norm=0.3134 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9765 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7507 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=18501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=18550 epoch=18550/19000 epoch_step=1/1 micro_steps=18550 elapsed=7.7s lr=2.000000e-03 loss=0.7985 loss_recon=0.7985 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6097 corrupt_frac=0.9977 acc_corrupt=0.6089 loss_corrupt=0.7985 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5347 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6775 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3188 out_g_norm=0.3275 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8646 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=18600 epoch=18600/19000 epoch_step=1/1 micro_steps=18600 elapsed=6.8s lr=2.000000e-03 loss=0.7961 loss_recon=0.7961 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6111 corrupt_frac=0.9977 acc_corrupt=0.6103 loss_corrupt=0.7961 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5331 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8107 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9046 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3184 out_g_norm=0.2743 acc_corrupt_t_0p8_1p0=0.9794 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8120 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=18650 epoch=18650/19000 epoch_step=1/1 micro_steps=18650 elapsed=6.8s lr=2.000000e-03 loss=0.7886 loss_recon=0.7886 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6113 corrupt_frac=0.9977 acc_corrupt=0.6104 loss_corrupt=0.7886 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5356 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8109 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9094 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3223 out_g_norm=0.3175 loss_all=0.8242 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=18700 epoch=18700/19000 epoch_step=1/1 micro_steps=18700 elapsed=6.7s lr=2.000000e-03 loss=0.8041 loss_recon=0.8041 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6089 corrupt_frac=0.9977 acc_corrupt=0.6080 loss_corrupt=0.8041 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5310 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6780 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8153 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9088 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3256 out_g_norm=0.3016 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7545 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=18750 epoch=18750/19000 epoch_step=1/1 micro_steps=18750 elapsed=6.7s lr=2.000000e-03 loss=0.7966 loss_recon=0.7966 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6114 corrupt_frac=0.9977 acc_corrupt=0.6105 loss_corrupt=0.7966 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5348 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6770 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8131 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3245 out_g_norm=0.3426 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8228 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=18800 epoch=18800/19000 epoch_step=1/1 micro_steps=18800 elapsed=6.7s lr=2.000000e-03 loss=0.7955 loss_recon=0.7955 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6117 corrupt_frac=0.9977 acc_corrupt=0.6108 loss_corrupt=0.7955 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5352 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6787 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3338 out_g_norm=0.3498 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7923 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=18850 epoch=18850/19000 epoch_step=1/1 micro_steps=18850 elapsed=6.7s lr=2.000000e-03 loss=0.8165 loss_recon=0.8165 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6060 corrupt_frac=0.9977 acc_corrupt=0.6051 loss_corrupt=0.8165 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5286 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8127 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3277 out_g_norm=0.3193 loss_all=0.8086 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=18900 epoch=18900/19000 epoch_step=1/1 micro_steps=18900 elapsed=6.7s lr=2.000000e-03 loss=0.7915 loss_recon=0.7915 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6124 corrupt_frac=0.9977 acc_corrupt=0.6116 loss_corrupt=0.7915 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8138 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3259 out_g_norm=0.2938 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7705 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=18950 epoch=18950/19000 epoch_step=1/1 micro_steps=18950 elapsed=6.7s lr=2.000000e-03 loss=0.7973 loss_recon=0.7973 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6107 corrupt_frac=0.9977 acc_corrupt=0.6098 loss_corrupt=0.7973 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5319 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3192 out_g_norm=0.2930 acc_corrupt_t_0p6_0p8=0.9113 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9745 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8078 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=19000 epoch=19000/19000 epoch_step=1/1 micro_steps=19000 elapsed=6.7s lr=2.000000e-03 loss=0.7994 loss_recon=0.7994 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6096 corrupt_frac=0.9977 acc_corrupt=0.6087 loss_corrupt=0.7994 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5339 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3134 out_g_norm=0.3020 acc_corrupt_t_0p6_0p8=0.9130 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9657 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7870 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=19001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=19050 epoch=19050/19500 epoch_step=1/1 micro_steps=19050 elapsed=7.6s lr=2.000000e-03 loss=0.7961 loss_recon=0.7961 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6102 corrupt_frac=0.9977 acc_corrupt=0.6093 loss_corrupt=0.7961 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5354 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6777 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8110 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3124 out_g_norm=0.2860 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8102 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=19100 epoch=19100/19500 epoch_step=1/1 micro_steps=19100 elapsed=6.6s lr=2.000000e-03 loss=0.7927 loss_recon=0.7927 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6117 corrupt_frac=0.9977 acc_corrupt=0.6108 loss_corrupt=0.7927 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5347 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9052 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3137 out_g_norm=0.3145 acc_corrupt_t_0p8_1p0=0.9843 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7618 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=19150 epoch=19150/19500 epoch_step=1/1 micro_steps=19150 elapsed=6.6s lr=2.000000e-03 loss=0.7917 loss_recon=0.7917 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6112 corrupt_frac=0.9977 acc_corrupt=0.6103 loss_corrupt=0.7917 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5355 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3122 out_g_norm=0.2946 loss_all=0.7883 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=19200 epoch=19200/19500 epoch_step=1/1 micro_steps=19200 elapsed=6.6s lr=2.000000e-03 loss=0.7989 loss_recon=0.7989 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6100 corrupt_frac=0.9977 acc_corrupt=0.6092 loss_corrupt=0.7989 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6780 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8153 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3180 out_g_norm=0.2922 acc_corrupt_t_0p8_1p0=0.9677 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7431 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=19250 epoch=19250/19500 epoch_step=1/1 micro_steps=19250 elapsed=6.7s lr=2.000000e-03 loss=0.7935 loss_recon=0.7935 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6111 corrupt_frac=0.9977 acc_corrupt=0.6102 loss_corrupt=0.7935 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5342 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6771 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8133 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3119 out_g_norm=0.2870 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7750 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=19300 epoch=19300/19500 epoch_step=1/1 micro_steps=19300 elapsed=6.6s lr=2.000000e-03 loss=0.7931 loss_recon=0.7931 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6114 corrupt_frac=0.9977 acc_corrupt=0.6105 loss_corrupt=0.7931 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5352 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6779 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9098 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3094 out_g_norm=0.3248 acc_corrupt_t_0p8_1p0=0.9834 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7395 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=19350 epoch=19350/19500 epoch_step=1/1 micro_steps=19350 elapsed=6.6s lr=2.000000e-03 loss=0.8049 loss_recon=0.8049 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6082 corrupt_frac=0.9977 acc_corrupt=0.6073 loss_corrupt=0.8049 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5325 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6757 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9102 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3048 out_g_norm=0.2763 loss_all=0.7850 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=19400 epoch=19400/19500 epoch_step=1/1 micro_steps=19400 elapsed=6.5s lr=2.000000e-03 loss=0.7843 loss_recon=0.7843 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6132 corrupt_frac=0.9977 acc_corrupt=0.6123 loss_corrupt=0.7843 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5358 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6773 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9108 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3024 out_g_norm=0.2845 acc_corrupt_t_0p8_1p0=0.9716 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7408 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=19450 epoch=19450/19500 epoch_step=1/1 micro_steps=19450 elapsed=6.5s lr=2.000000e-03 loss=0.7985 loss_recon=0.7985 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6111 corrupt_frac=0.9977 acc_corrupt=0.6102 loss_corrupt=0.7985 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5324 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8137 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3108 out_g_norm=0.3169 acc_corrupt_t_0p6_0p8=0.9130 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9770 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8127 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=19500 epoch=19500/19500 epoch_step=1/1 micro_steps=19500 elapsed=6.5s lr=2.000000e-03 loss=0.7973 loss_recon=0.7973 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6100 corrupt_frac=0.9977 acc_corrupt=0.6091 loss_corrupt=0.7973 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5346 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6768 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8115 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.3047 out_g_norm=0.3053 acc_corrupt_t_0p6_0p8=0.9123 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9775 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7503 init_gold_top10=0.2037 init_gold_top100=0.3252 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt start_step=19501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 20000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n1024_compactv2664_3l_bs512_hard_ce_onehot/latest.pt" +} +step=19550 epoch=19550/20000 epoch_step=1/1 micro_steps=19550 elapsed=7.6s lr=2.000000e-03 loss=0.7894 loss_recon=0.7894 loss_meanflow=0.0000 mean_model_t=0.2080 mean_corrupt_t=0.2080 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6112 corrupt_frac=0.9977 acc_corrupt=0.6103 loss_corrupt=0.7894 wrong_frac=0.7919 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.5368 corrupt_frac_t_0p0_0p2=0.5673 acc_corrupt_t_0p2_0p4=0.6783 corrupt_frac_t_0p2_0p4=0.3464 acc_corrupt_t_0p4_0p6=0.8112 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9093 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3087 out_g_norm=0.2695 acc_corrupt_t_0p8_1p0=0.9809 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7736 init_gold_top10=0.1849 init_gold_top100=0.3118 +step=19600 epoch=19600/20000 epoch_step=1/1 micro_steps=19600 elapsed=6.6s lr=2.000000e-03 loss=0.7892 loss_recon=0.7892 loss_meanflow=0.0000 mean_model_t=0.2100 mean_corrupt_t=0.2100 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6123 corrupt_frac=0.9977 acc_corrupt=0.6114 loss_corrupt=0.7892 wrong_frac=0.7900 init_acc_corrupt=0.1173 acc_corrupt_t_0p0_0p2=0.5352 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6776 corrupt_frac_t_0p2_0p4=0.3612 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0797 acc_corrupt_t_0p6_0p8=0.9050 corrupt_frac_t_0p6_0p8=0.0115 out_w_norm=11.3170 out_g_norm=0.3114 acc_corrupt_t_0p8_1p0=0.9821 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7778 init_gold_top10=0.1940 init_gold_top100=0.3207 +step=19650 epoch=19650/20000 epoch_step=1/1 micro_steps=19650 elapsed=6.5s lr=2.000000e-03 loss=0.7892 loss_recon=0.7892 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6116 corrupt_frac=0.9977 acc_corrupt=0.6107 loss_corrupt=0.7892 wrong_frac=0.7913 init_acc_corrupt=0.1155 acc_corrupt_t_0p0_0p2=0.5360 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.8111 corrupt_frac_t_0p4_0p6=0.0783 acc_corrupt_t_0p6_0p8=0.9096 corrupt_frac_t_0p6_0p8=0.0118 out_w_norm=11.3126 out_g_norm=0.3118 loss_all=0.8025 init_gold_top10=0.1932 init_gold_top100=0.3210 +step=19700 epoch=19700/20000 epoch_step=1/1 micro_steps=19700 elapsed=6.5s lr=2.000000e-03 loss=0.7942 loss_recon=0.7942 loss_meanflow=0.0000 mean_model_t=0.2086 mean_corrupt_t=0.2086 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6105 corrupt_frac=0.9977 acc_corrupt=0.6096 loss_corrupt=0.7942 wrong_frac=0.7912 init_acc_corrupt=0.1162 acc_corrupt_t_0p0_0p2=0.5337 corrupt_frac_t_0p0_0p2=0.5609 acc_corrupt_t_0p2_0p4=0.6783 corrupt_frac_t_0p2_0p4=0.3534 acc_corrupt_t_0p4_0p6=0.8147 corrupt_frac_t_0p4_0p6=0.0778 acc_corrupt_t_0p6_0p8=0.9083 corrupt_frac_t_0p6_0p8=0.0106 out_w_norm=11.3135 out_g_norm=0.3116 acc_corrupt_t_0p8_1p0=0.9736 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7589 init_gold_top10=0.2029 init_gold_top100=0.3244 +step=19750 epoch=19750/20000 epoch_step=1/1 micro_steps=19750 elapsed=6.5s lr=2.000000e-03 loss=0.7981 loss_recon=0.7981 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6106 corrupt_frac=0.9977 acc_corrupt=0.6098 loss_corrupt=0.7981 wrong_frac=0.7901 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5339 corrupt_frac_t_0p0_0p2=0.5589 acc_corrupt_t_0p2_0p4=0.6762 corrupt_frac_t_0p2_0p4=0.3523 acc_corrupt_t_0p4_0p6=0.8130 corrupt_frac_t_0p4_0p6=0.0792 out_w_norm=11.3053 out_g_norm=0.3393 acc_corrupt_t_0p6_0p8=0.9095 corrupt_frac_t_0p6_0p8=0.0138 acc_corrupt_t_0p8_1p0=0.9833 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8233 init_gold_top10=0.1978 init_gold_top100=0.3220 +step=19800 epoch=19800/20000 epoch_step=1/1 micro_steps=19800 elapsed=6.5s lr=2.000000e-03 loss=0.8011 loss_recon=0.8011 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6103 corrupt_frac=0.9977 acc_corrupt=0.6094 loss_corrupt=0.8011 wrong_frac=0.7906 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.5333 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6778 corrupt_frac_t_0p2_0p4=0.3598 acc_corrupt_t_0p4_0p6=0.8116 corrupt_frac_t_0p4_0p6=0.0748 acc_corrupt_t_0p6_0p8=0.9101 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.3004 out_g_norm=0.2944 acc_corrupt_t_0p8_1p0=0.9795 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7918 init_gold_top10=0.2159 init_gold_top100=0.3288 +step=19850 epoch=19850/20000 epoch_step=1/1 micro_steps=19850 elapsed=6.5s lr=2.000000e-03 loss=0.8028 loss_recon=0.8028 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6086 corrupt_frac=0.9977 acc_corrupt=0.6077 loss_corrupt=0.8028 wrong_frac=0.7933 init_acc_corrupt=0.1128 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5581 acc_corrupt_t_0p2_0p4=0.6759 corrupt_frac_t_0p2_0p4=0.3639 acc_corrupt_t_0p4_0p6=0.8132 corrupt_frac_t_0p4_0p6=0.0689 acc_corrupt_t_0p6_0p8=0.9107 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=11.3003 out_g_norm=0.3125 loss_all=0.7859 init_gold_top10=0.1829 init_gold_top100=0.3157 +step=19900 epoch=19900/20000 epoch_step=1/1 micro_steps=19900 elapsed=6.5s lr=2.000000e-03 loss=0.7814 loss_recon=0.7814 loss_meanflow=0.0000 mean_model_t=0.2111 mean_corrupt_t=0.2111 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6136 corrupt_frac=0.9977 acc_corrupt=0.6128 loss_corrupt=0.7814 wrong_frac=0.7888 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5368 corrupt_frac_t_0p0_0p2=0.5500 acc_corrupt_t_0p2_0p4=0.6769 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.8144 corrupt_frac_t_0p4_0p6=0.0792 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.3031 out_g_norm=0.3042 acc_corrupt_t_0p8_1p0=0.9672 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7548 init_gold_top10=0.1967 init_gold_top100=0.3221 +step=19950 epoch=19950/20000 epoch_step=1/1 micro_steps=19950 elapsed=6.5s lr=2.000000e-03 loss=0.7975 loss_recon=0.7975 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6111 corrupt_frac=0.9977 acc_corrupt=0.6102 loss_corrupt=0.7975 wrong_frac=0.7898 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.5327 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.6772 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8136 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.3011 out_g_norm=0.3081 acc_corrupt_t_0p6_0p8=0.9118 corrupt_frac_t_0p6_0p8=0.0110 acc_corrupt_t_0p8_1p0=0.9780 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7996 init_gold_top10=0.1874 init_gold_top100=0.3179 +step=20000 epoch=20000/20000 epoch_step=1/1 micro_steps=20000 elapsed=6.5s lr=2.000000e-03 loss=0.7967 loss_recon=0.7967 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6099 corrupt_frac=0.9977 acc_corrupt=0.6090 loss_corrupt=0.7967 wrong_frac=0.7921 init_acc_corrupt=0.1149 acc_corrupt_t_0p0_0p2=0.5344 corrupt_frac_t_0p0_0p2=0.5623 acc_corrupt_t_0p2_0p4=0.6767 corrupt_frac_t_0p2_0p4=0.3512 acc_corrupt_t_0p4_0p6=0.8117 corrupt_frac_t_0p4_0p6=0.0802 out_w_norm=11.2970 out_g_norm=0.2954 acc_corrupt_t_0p6_0p8=0.9124 corrupt_frac_t_0p6_0p8=0.0105 acc_corrupt_t_0p8_1p0=0.9735 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7451 init_gold_top10=0.2037 init_gold_top100=0.3252 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt.log new file mode 100644 index 0000000000000000000000000000000000000000..e6fb44d9ebea97d772b41d71d343e30376b87134 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt.log @@ -0,0 +1,7919 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=50 epoch=50/500 epoch_step=1/1 micro_steps=50 elapsed=3.7s lr=2.000000e-03 loss=7.3432 loss_recon=7.3432 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0938 corrupt_frac=1.0000 acc_corrupt=0.0938 loss_corrupt=7.3432 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.0463 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.1273 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.2683 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.4123 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=0.4999 out_g_norm=0.7797 acc_corrupt_t_0p8_1p0=0.5391 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.2481 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=3.0s lr=2.000000e-03 loss=7.1260 loss_recon=7.1260 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0982 corrupt_frac=1.0000 acc_corrupt=0.0982 loss_corrupt=7.1260 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.0528 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.1296 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.2445 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=1.5958 out_g_norm=1.3020 acc_corrupt_t_0p6_0p8=0.3563 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.4375 corrupt_frac_t_0p8_1p0=0.0078 loss_all=7.0080 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=150 epoch=150/500 epoch_step=1/1 micro_steps=150 elapsed=3.0s lr=2.000000e-03 loss=6.8071 loss_recon=6.8071 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1011 corrupt_frac=1.0000 acc_corrupt=0.1011 loss_corrupt=6.8071 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.0539 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.1385 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.2597 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.3586 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=2.5652 out_g_norm=1.6201 acc_corrupt_t_0p8_1p0=0.4967 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.6679 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=3.0s lr=2.000000e-03 loss=6.4305 loss_recon=6.4305 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1085 corrupt_frac=1.0000 acc_corrupt=0.1085 loss_corrupt=6.4305 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.0560 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.1472 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.2716 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=3.5047 out_g_norm=1.6061 acc_corrupt_t_0p6_0p8=0.3883 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.4766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.3339 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=250 epoch=250/500 epoch_step=1/1 micro_steps=250 elapsed=3.0s lr=2.000000e-03 loss=6.1384 loss_recon=6.1384 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1150 corrupt_frac=1.0000 acc_corrupt=0.1150 loss_corrupt=6.1384 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.0578 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.1546 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.2926 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.4197 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=4.3400 out_g_norm=1.2489 acc_corrupt_t_0p8_1p0=0.5273 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.9984 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=3.0s lr=2.000000e-03 loss=5.8806 loss_recon=5.8806 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1225 corrupt_frac=1.0000 acc_corrupt=0.1225 loss_corrupt=5.8806 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.0600 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.1652 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.3127 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.4362 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=4.9941 out_g_norm=0.7780 loss_all=5.7527 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=350 epoch=350/500 epoch_step=1/1 micro_steps=350 elapsed=3.0s lr=2.000000e-03 loss=5.6806 loss_recon=5.6806 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1247 corrupt_frac=1.0000 acc_corrupt=0.1247 loss_corrupt=5.6806 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.0623 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.1745 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.3314 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.4660 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=5.7245 out_g_norm=0.4580 loss_all=5.4848 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=3.0s lr=2.000000e-03 loss=5.4226 loss_recon=5.4226 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1338 corrupt_frac=1.0000 acc_corrupt=0.1338 loss_corrupt=5.4226 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.0638 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.1870 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.3558 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.4962 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=6.4671 out_g_norm=0.3610 loss_all=5.3166 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=450 epoch=450/500 epoch_step=1/1 micro_steps=450 elapsed=3.0s lr=2.000000e-03 loss=5.1362 loss_recon=5.1362 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1439 corrupt_frac=1.0000 acc_corrupt=0.1439 loss_corrupt=5.1362 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.0662 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.1987 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.3819 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=7.1724 out_g_norm=0.3895 acc_corrupt_t_0p6_0p8=0.5663 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.7598 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.0691 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=3.0s lr=2.000000e-03 loss=4.8201 loss_recon=4.8201 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1562 corrupt_frac=1.0000 acc_corrupt=0.1562 loss_corrupt=4.8201 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.0691 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.2164 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.4490 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.6735 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=7.8932 out_g_norm=0.4326 acc_corrupt_t_0p8_1p0=0.8242 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.6206 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=550 epoch=550/1000 epoch_step=1/1 micro_steps=550 elapsed=3.7s lr=2.000000e-03 loss=4.4899 loss_recon=4.4899 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1701 corrupt_frac=1.0000 acc_corrupt=0.1701 loss_corrupt=4.4899 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.0736 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.2451 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.5027 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.6882 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=8.5778 out_g_norm=0.4304 acc_corrupt_t_0p8_1p0=0.8516 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.1519 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.0s lr=2.000000e-03 loss=4.1080 loss_recon=4.1080 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1905 corrupt_frac=1.0000 acc_corrupt=0.1905 loss_corrupt=4.1080 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.0779 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.2796 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.5178 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.2063 out_g_norm=0.4418 acc_corrupt_t_0p6_0p8=0.6856 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.8848 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.0605 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=650 epoch=650/1000 epoch_step=1/1 micro_steps=650 elapsed=3.0s lr=2.000000e-03 loss=3.8450 loss_recon=3.8450 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1995 corrupt_frac=1.0000 acc_corrupt=0.1995 loss_corrupt=3.8450 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.0826 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.3045 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.5473 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.6989 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.7410 out_g_norm=0.5041 acc_corrupt_t_0p8_1p0=0.8542 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.0690 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.0s lr=2.000000e-03 loss=3.5223 loss_recon=3.5223 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2212 corrupt_frac=1.0000 acc_corrupt=0.2212 loss_corrupt=3.5223 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.0906 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.3324 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.5761 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=10.1515 out_g_norm=0.6294 acc_corrupt_t_0p6_0p8=0.7282 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8828 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.5259 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=750 epoch=750/1000 epoch_step=1/1 micro_steps=750 elapsed=3.0s lr=2.000000e-03 loss=3.2378 loss_recon=3.2378 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2452 corrupt_frac=1.0000 acc_corrupt=0.2452 loss_corrupt=3.2378 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.0996 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.3710 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.6168 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.7585 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=10.4506 out_g_norm=0.7335 acc_corrupt_t_0p8_1p0=0.8428 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.2037 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.0s lr=2.000000e-03 loss=2.9874 loss_recon=2.9874 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2719 corrupt_frac=1.0000 acc_corrupt=0.2719 loss_corrupt=2.9874 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.1113 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.4130 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.6595 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.7901 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=10.6670 out_g_norm=0.7425 loss_all=2.9043 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=850 epoch=850/1000 epoch_step=1/1 micro_steps=850 elapsed=3.0s lr=2.000000e-03 loss=2.7937 loss_recon=2.7937 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2934 corrupt_frac=1.0000 acc_corrupt=0.2934 loss_corrupt=2.7937 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.1298 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.4600 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.6983 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.8184 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=10.8312 out_g_norm=0.8605 loss_all=2.5470 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.0s lr=2.000000e-03 loss=2.5101 loss_recon=2.5101 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3399 corrupt_frac=1.0000 acc_corrupt=0.3399 loss_corrupt=2.5101 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.1630 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.5187 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.7408 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.8437 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=10.9505 out_g_norm=0.9768 loss_all=2.4354 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=950 epoch=950/1000 epoch_step=1/1 micro_steps=950 elapsed=3.0s lr=2.000000e-03 loss=2.1534 loss_recon=2.1534 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4104 corrupt_frac=1.0000 acc_corrupt=0.4104 loss_corrupt=2.1534 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.2241 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.6014 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7859 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.0490 out_g_norm=1.1254 acc_corrupt_t_0p6_0p8=0.8602 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9414 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.0624 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.0s lr=2.000000e-03 loss=1.7743 loss_recon=1.7743 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4871 corrupt_frac=1.0000 acc_corrupt=0.4871 loss_corrupt=1.7743 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.3035 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6914 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8230 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.8875 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.1456 out_g_norm=1.2021 acc_corrupt_t_0p8_1p0=0.9463 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4695 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1050 epoch=1050/1500 epoch_step=1/1 micro_steps=1050 elapsed=3.7s lr=2.000000e-03 loss=1.4996 loss_recon=1.4996 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5393 corrupt_frac=1.0000 acc_corrupt=0.5393 loss_corrupt=1.4996 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.3667 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.7450 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8476 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9006 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.2433 out_g_norm=1.3162 acc_corrupt_t_0p8_1p0=0.9590 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2403 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=1100 epoch=1100/1500 epoch_step=1/1 micro_steps=1100 elapsed=3.0s lr=2.000000e-03 loss=1.3050 loss_recon=1.3050 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5762 corrupt_frac=1.0000 acc_corrupt=0.5762 loss_corrupt=1.3050 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.3958 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.7816 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8601 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.3264 out_g_norm=1.2340 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9395 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3052 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=1150 epoch=1150/1500 epoch_step=1/1 micro_steps=1150 elapsed=3.0s lr=2.000000e-03 loss=1.2360 loss_recon=1.2360 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5841 corrupt_frac=1.0000 acc_corrupt=0.5841 loss_corrupt=1.2360 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.4123 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.7931 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8668 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9109 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.3731 out_g_norm=1.3370 acc_corrupt_t_0p8_1p0=0.9460 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4774 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=1200 epoch=1200/1500 epoch_step=1/1 micro_steps=1200 elapsed=3.0s lr=2.000000e-03 loss=1.1434 loss_recon=1.1434 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6044 corrupt_frac=1.0000 acc_corrupt=0.6044 loss_corrupt=1.1434 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.4324 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8015 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8694 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.3820 out_g_norm=1.1982 acc_corrupt_t_0p6_0p8=0.9121 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9629 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1908 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=1250 epoch=1250/1500 epoch_step=1/1 micro_steps=1250 elapsed=3.0s lr=2.000000e-03 loss=1.0902 loss_recon=1.0902 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6159 corrupt_frac=1.0000 acc_corrupt=0.6159 loss_corrupt=1.0902 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.4475 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8072 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8705 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9165 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.3881 out_g_norm=1.0849 acc_corrupt_t_0p8_1p0=0.9463 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2190 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=1300 epoch=1300/1500 epoch_step=1/1 micro_steps=1300 elapsed=3.0s lr=2.000000e-03 loss=1.0687 loss_recon=1.0687 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6208 corrupt_frac=1.0000 acc_corrupt=0.6208 loss_corrupt=1.0687 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.4536 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8097 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8731 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9162 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.3809 out_g_norm=1.0798 loss_all=1.1876 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=1350 epoch=1350/1500 epoch_step=1/1 micro_steps=1350 elapsed=3.0s lr=2.000000e-03 loss=1.0613 loss_recon=1.0613 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6194 corrupt_frac=1.0000 acc_corrupt=0.6194 loss_corrupt=1.0613 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.4629 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8107 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8707 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9146 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.3800 out_g_norm=1.0474 loss_all=1.0609 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=1400 epoch=1400/1500 epoch_step=1/1 micro_steps=1400 elapsed=3.0s lr=2.000000e-03 loss=1.0288 loss_recon=1.0288 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6285 corrupt_frac=1.0000 acc_corrupt=0.6285 loss_corrupt=1.0288 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.4726 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8126 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8739 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9208 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.3761 out_g_norm=1.0102 loss_all=1.0958 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=1450 epoch=1450/1500 epoch_step=1/1 micro_steps=1450 elapsed=3.0s lr=2.000000e-03 loss=1.0033 loss_recon=1.0033 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6340 corrupt_frac=1.0000 acc_corrupt=0.6340 loss_corrupt=1.0033 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.4771 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8137 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8751 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.3774 out_g_norm=0.9682 acc_corrupt_t_0p6_0p8=0.9185 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9668 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0537 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=1500 epoch=1500/1500 epoch_step=1/1 micro_steps=1500 elapsed=3.0s lr=2.000000e-03 loss=0.9785 loss_recon=0.9785 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6393 corrupt_frac=1.0000 acc_corrupt=0.6393 loss_corrupt=0.9785 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.4895 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8149 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8756 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9248 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.3780 out_g_norm=0.9499 acc_corrupt_t_0p8_1p0=0.9590 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8492 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1550 epoch=1550/2000 epoch_step=1/1 micro_steps=1550 elapsed=3.7s lr=2.000000e-03 loss=0.9759 loss_recon=0.9759 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6410 corrupt_frac=1.0000 acc_corrupt=0.6410 loss_corrupt=0.9759 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.4983 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8155 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8763 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9237 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.3789 out_g_norm=0.8411 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8671 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=3.1s lr=2.000000e-03 loss=0.9419 loss_recon=0.9419 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6491 corrupt_frac=1.0000 acc_corrupt=0.6491 loss_corrupt=0.9419 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.5019 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8177 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8757 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.3820 out_g_norm=0.7979 acc_corrupt_t_0p6_0p8=0.9217 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9766 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=1650 epoch=1650/2000 epoch_step=1/1 micro_steps=1650 elapsed=3.2s lr=2.000000e-03 loss=0.9606 loss_recon=0.9606 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6427 corrupt_frac=1.0000 acc_corrupt=0.6427 loss_corrupt=0.9606 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.4985 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8180 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8794 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9233 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.3820 out_g_norm=0.8496 acc_corrupt_t_0p8_1p0=0.9538 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2620 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=3.2s lr=2.000000e-03 loss=0.9115 loss_recon=0.9115 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6571 corrupt_frac=1.0000 acc_corrupt=0.6571 loss_corrupt=0.9115 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.5149 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8190 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8802 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.3779 out_g_norm=0.7370 acc_corrupt_t_0p6_0p8=0.9242 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9805 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=1750 epoch=1750/2000 epoch_step=1/1 micro_steps=1750 elapsed=3.2s lr=2.000000e-03 loss=0.9009 loss_recon=0.9009 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6598 corrupt_frac=1.0000 acc_corrupt=0.6598 loss_corrupt=0.9009 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.5173 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8201 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8807 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9271 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.3913 out_g_norm=0.7194 acc_corrupt_t_0p8_1p0=0.9580 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0139 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=3.2s lr=2.000000e-03 loss=0.8900 loss_recon=0.8900 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6642 corrupt_frac=1.0000 acc_corrupt=0.6642 loss_corrupt=0.8900 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.5237 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8211 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.8826 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9277 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.3922 out_g_norm=0.8237 loss_all=1.0765 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=1850 epoch=1850/2000 epoch_step=1/1 micro_steps=1850 elapsed=3.2s lr=2.000000e-03 loss=0.9035 loss_recon=0.9035 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6601 corrupt_frac=1.0000 acc_corrupt=0.6601 loss_corrupt=0.9035 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.5263 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8218 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.8825 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9267 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.3971 out_g_norm=0.7431 loss_all=0.9334 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=3.2s lr=2.000000e-03 loss=0.8751 loss_recon=0.8751 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6668 corrupt_frac=1.0000 acc_corrupt=0.6668 loss_corrupt=0.8751 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.5330 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8228 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.8853 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9322 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.4076 out_g_norm=0.6623 loss_all=0.8978 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=1950 epoch=1950/2000 epoch_step=1/1 micro_steps=1950 elapsed=3.1s lr=2.000000e-03 loss=0.8691 loss_recon=0.8691 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6689 corrupt_frac=1.0000 acc_corrupt=0.6689 loss_corrupt=0.8691 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.5319 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8234 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.8877 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.4133 out_g_norm=0.7276 acc_corrupt_t_0p6_0p8=0.9329 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8957 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=3.0s lr=2.000000e-03 loss=0.8546 loss_recon=0.8546 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6728 corrupt_frac=1.0000 acc_corrupt=0.6728 loss_corrupt=0.8546 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.5416 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8240 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8892 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9377 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.4206 out_g_norm=0.6807 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7603 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2050 epoch=2050/2500 epoch_step=1/1 micro_steps=2050 elapsed=3.7s lr=2.000000e-03 loss=0.8401 loss_recon=0.8401 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6762 corrupt_frac=1.0000 acc_corrupt=0.6762 loss_corrupt=0.8401 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.5523 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8249 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.8912 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9386 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.4314 out_g_norm=0.6466 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8291 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=2100 epoch=2100/2500 epoch_step=1/1 micro_steps=2100 elapsed=3.0s lr=2.000000e-03 loss=0.8181 loss_recon=0.8181 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6843 corrupt_frac=1.0000 acc_corrupt=0.6843 loss_corrupt=0.8181 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.5569 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8276 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.8916 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.4471 out_g_norm=0.6765 acc_corrupt_t_0p6_0p8=0.9406 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9434 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8610 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=2150 epoch=2150/2500 epoch_step=1/1 micro_steps=2150 elapsed=3.0s lr=2.000000e-03 loss=0.8334 loss_recon=0.8334 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6793 corrupt_frac=1.0000 acc_corrupt=0.6793 loss_corrupt=0.8334 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.5551 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8276 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.8957 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9395 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.4552 out_g_norm=0.6777 acc_corrupt_t_0p8_1p0=0.9694 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0992 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=2200 epoch=2200/2500 epoch_step=1/1 micro_steps=2200 elapsed=3.0s lr=2.000000e-03 loss=0.7982 loss_recon=0.7982 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6897 corrupt_frac=1.0000 acc_corrupt=0.6897 loss_corrupt=0.7982 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.5647 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8291 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.8977 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.4588 out_g_norm=0.6641 acc_corrupt_t_0p6_0p8=0.9429 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9842 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=2250 epoch=2250/2500 epoch_step=1/1 micro_steps=2250 elapsed=3.0s lr=2.000000e-03 loss=0.7964 loss_recon=0.7964 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6921 corrupt_frac=1.0000 acc_corrupt=0.6921 loss_corrupt=0.7964 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.5666 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8300 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.8987 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9459 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.4634 out_g_norm=0.7258 acc_corrupt_t_0p8_1p0=0.9717 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9006 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=2300 epoch=2300/2500 epoch_step=1/1 micro_steps=2300 elapsed=3.0s lr=2.000000e-03 loss=0.7783 loss_recon=0.7783 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6972 corrupt_frac=1.0000 acc_corrupt=0.6972 loss_corrupt=0.7783 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.5735 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8315 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9031 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9481 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.4725 out_g_norm=0.6109 loss_all=0.9389 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=2350 epoch=2350/2500 epoch_step=1/1 micro_steps=2350 elapsed=3.0s lr=2.000000e-03 loss=0.8007 loss_recon=0.8007 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6905 corrupt_frac=1.0000 acc_corrupt=0.6905 loss_corrupt=0.8007 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.5705 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8325 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9032 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9477 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.4853 out_g_norm=0.5730 loss_all=0.7977 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=2400 epoch=2400/2500 epoch_step=1/1 micro_steps=2400 elapsed=3.0s lr=2.000000e-03 loss=0.7810 loss_recon=0.7810 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6970 corrupt_frac=1.0000 acc_corrupt=0.6970 loss_corrupt=0.7810 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.5771 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8336 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9055 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9529 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.4910 out_g_norm=0.5707 loss_all=0.7898 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=2450 epoch=2450/2500 epoch_step=1/1 micro_steps=2450 elapsed=3.0s lr=2.000000e-03 loss=0.7766 loss_recon=0.7766 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6977 corrupt_frac=1.0000 acc_corrupt=0.6977 loss_corrupt=0.7766 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.5746 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8334 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9077 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.4894 out_g_norm=0.5574 acc_corrupt_t_0p6_0p8=0.9530 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7951 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=2500 epoch=2500/2500 epoch_step=1/1 micro_steps=2500 elapsed=3.0s lr=2.000000e-03 loss=0.7565 loss_recon=0.7565 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7036 corrupt_frac=1.0000 acc_corrupt=0.7036 loss_corrupt=0.7565 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.5874 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8344 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9083 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9563 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.4945 out_g_norm=0.5456 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6915 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2550 epoch=2550/3000 epoch_step=1/1 micro_steps=2550 elapsed=3.7s lr=2.000000e-03 loss=0.7615 loss_recon=0.7615 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7008 corrupt_frac=1.0000 acc_corrupt=0.7008 loss_corrupt=0.7615 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.5874 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8343 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9090 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9576 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.5088 out_g_norm=0.5506 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7605 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=3.0s lr=2.000000e-03 loss=0.7295 loss_recon=0.7295 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7111 corrupt_frac=1.0000 acc_corrupt=0.7111 loss_corrupt=0.7295 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.5968 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8366 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9089 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.5224 out_g_norm=0.5278 acc_corrupt_t_0p6_0p8=0.9576 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7769 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=2650 epoch=2650/3000 epoch_step=1/1 micro_steps=2650 elapsed=3.0s lr=2.000000e-03 loss=0.7583 loss_recon=0.7583 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7030 corrupt_frac=1.0000 acc_corrupt=0.7030 loss_corrupt=0.7583 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.5895 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8361 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9109 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9563 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.5300 out_g_norm=0.6592 acc_corrupt_t_0p8_1p0=0.9857 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9978 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=3.0s lr=2.000000e-03 loss=0.7208 loss_recon=0.7208 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7142 corrupt_frac=1.0000 acc_corrupt=0.7142 loss_corrupt=0.7208 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6019 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8369 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9117 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.5367 out_g_norm=0.5791 acc_corrupt_t_0p6_0p8=0.9563 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8575 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=2750 epoch=2750/3000 epoch_step=1/1 micro_steps=2750 elapsed=3.0s lr=2.000000e-03 loss=0.7263 loss_recon=0.7263 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7137 corrupt_frac=1.0000 acc_corrupt=0.7137 loss_corrupt=0.7263 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.5997 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8368 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9109 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9597 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.5444 out_g_norm=0.6229 acc_corrupt_t_0p8_1p0=0.9814 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7871 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=3.0s lr=2.000000e-03 loss=0.7173 loss_recon=0.7173 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7171 corrupt_frac=1.0000 acc_corrupt=0.7171 loss_corrupt=0.7173 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6046 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8370 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9130 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9579 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.5459 out_g_norm=0.5713 loss_all=0.8080 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=2850 epoch=2850/3000 epoch_step=1/1 micro_steps=2850 elapsed=3.0s lr=2.000000e-03 loss=0.7257 loss_recon=0.7257 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7123 corrupt_frac=1.0000 acc_corrupt=0.7123 loss_corrupt=0.7257 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6046 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8380 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9112 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9559 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.5554 out_g_norm=0.5330 loss_all=0.7507 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=3.0s lr=2.000000e-03 loss=0.7171 loss_recon=0.7171 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7157 corrupt_frac=1.0000 acc_corrupt=0.7157 loss_corrupt=0.7171 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6068 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8380 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9128 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.5635 out_g_norm=0.5508 loss_all=0.6477 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=2950 epoch=2950/3000 epoch_step=1/1 micro_steps=2950 elapsed=3.0s lr=2.000000e-03 loss=0.7052 loss_recon=0.7052 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7176 corrupt_frac=1.0000 acc_corrupt=0.7176 loss_corrupt=0.7052 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6064 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8384 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9140 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.5744 out_g_norm=0.4751 acc_corrupt_t_0p6_0p8=0.9590 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7229 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=3.0s lr=2.000000e-03 loss=0.6930 loss_recon=0.6930 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7204 corrupt_frac=1.0000 acc_corrupt=0.7204 loss_corrupt=0.6930 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6144 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8383 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9135 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9611 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.5849 out_g_norm=0.4775 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6123 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3050 epoch=3050/3500 epoch_step=1/1 micro_steps=3050 elapsed=3.7s lr=2.000000e-03 loss=0.7049 loss_recon=0.7049 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7183 corrupt_frac=1.0000 acc_corrupt=0.7183 loss_corrupt=0.7049 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6157 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8373 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9134 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.5959 out_g_norm=0.4987 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6856 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=3100 epoch=3100/3500 epoch_step=1/1 micro_steps=3100 elapsed=3.0s lr=2.000000e-03 loss=0.6747 loss_recon=0.6747 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7268 corrupt_frac=1.0000 acc_corrupt=0.7268 loss_corrupt=0.6747 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6229 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8394 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9127 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6068 out_g_norm=0.5079 acc_corrupt_t_0p6_0p8=0.9597 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7659 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=3150 epoch=3150/3500 epoch_step=1/1 micro_steps=3150 elapsed=3.0s lr=2.000000e-03 loss=0.7068 loss_recon=0.7068 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7185 corrupt_frac=1.0000 acc_corrupt=0.7185 loss_corrupt=0.7068 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6151 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8385 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9133 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9586 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6106 out_g_norm=0.5809 acc_corrupt_t_0p8_1p0=0.9870 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9610 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=3200 epoch=3200/3500 epoch_step=1/1 micro_steps=3200 elapsed=3.0s lr=2.000000e-03 loss=0.6681 loss_recon=0.6681 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7297 corrupt_frac=1.0000 acc_corrupt=0.7297 loss_corrupt=0.6681 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6284 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8389 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9140 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6090 out_g_norm=0.4545 acc_corrupt_t_0p6_0p8=0.9587 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7788 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=3250 epoch=3250/3500 epoch_step=1/1 micro_steps=3250 elapsed=3.0s lr=2.000000e-03 loss=0.6619 loss_recon=0.6619 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7316 corrupt_frac=1.0000 acc_corrupt=0.7316 loss_corrupt=0.6619 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6301 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8395 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9128 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6152 out_g_norm=0.4695 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7022 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=3300 epoch=3300/3500 epoch_step=1/1 micro_steps=3300 elapsed=3.0s lr=2.000000e-03 loss=0.6679 loss_recon=0.6679 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7296 corrupt_frac=1.0000 acc_corrupt=0.7296 loss_corrupt=0.6679 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6258 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8389 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9605 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6241 out_g_norm=0.5177 loss_all=0.7049 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=3350 epoch=3350/3500 epoch_step=1/1 micro_steps=3350 elapsed=3.0s lr=2.000000e-03 loss=0.6677 loss_recon=0.6677 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7288 corrupt_frac=1.0000 acc_corrupt=0.7288 loss_corrupt=0.6677 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6325 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8397 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9124 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9578 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6329 out_g_norm=0.5197 loss_all=0.6900 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=3400 epoch=3400/3500 epoch_step=1/1 micro_steps=3400 elapsed=3.0s lr=2.000000e-03 loss=0.6685 loss_recon=0.6685 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7294 corrupt_frac=1.0000 acc_corrupt=0.7294 loss_corrupt=0.6685 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6302 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8395 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6348 out_g_norm=0.5479 loss_all=0.6543 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=3450 epoch=3450/3500 epoch_step=1/1 micro_steps=3450 elapsed=3.0s lr=2.000000e-03 loss=0.6459 loss_recon=0.6459 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7345 corrupt_frac=1.0000 acc_corrupt=0.7345 loss_corrupt=0.6459 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6361 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8396 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6438 out_g_norm=0.4665 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5890 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=3500 epoch=3500/3500 epoch_step=1/1 micro_steps=3500 elapsed=3.0s lr=2.000000e-03 loss=0.6419 loss_recon=0.6419 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7362 corrupt_frac=1.0000 acc_corrupt=0.7362 loss_corrupt=0.6419 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6421 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8392 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9140 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.6507 out_g_norm=0.5674 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5403 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3550 epoch=3550/4000 epoch_step=1/1 micro_steps=3550 elapsed=3.9s lr=2.000000e-03 loss=0.6581 loss_recon=0.6581 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7322 corrupt_frac=1.0000 acc_corrupt=0.7322 loss_corrupt=0.6581 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6398 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8382 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6587 out_g_norm=0.5655 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6073 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=3.2s lr=2.000000e-03 loss=0.6245 loss_recon=0.6245 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7412 corrupt_frac=1.0000 acc_corrupt=0.7412 loss_corrupt=0.6245 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6478 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9133 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.6681 out_g_norm=0.5280 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7250 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=3650 epoch=3650/4000 epoch_step=1/1 micro_steps=3650 elapsed=3.2s lr=2.000000e-03 loss=0.6573 loss_recon=0.6573 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7310 corrupt_frac=1.0000 acc_corrupt=0.7310 loss_corrupt=0.6573 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6365 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8393 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9135 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.6775 out_g_norm=0.4921 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8576 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=3.2s lr=2.000000e-03 loss=0.6431 loss_recon=0.6431 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7369 corrupt_frac=1.0000 acc_corrupt=0.7369 loss_corrupt=0.6431 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6409 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8397 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9140 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.6749 out_g_norm=0.4673 acc_corrupt_t_0p6_0p8=0.9599 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7635 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=3750 epoch=3750/4000 epoch_step=1/1 micro_steps=3750 elapsed=3.2s lr=2.000000e-03 loss=0.6218 loss_recon=0.6218 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7424 corrupt_frac=1.0000 acc_corrupt=0.7424 loss_corrupt=0.6218 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6496 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8398 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9134 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9630 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.6741 out_g_norm=0.4908 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6893 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=3.2s lr=2.000000e-03 loss=0.6337 loss_recon=0.6337 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7391 corrupt_frac=1.0000 acc_corrupt=0.7391 loss_corrupt=0.6337 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6428 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8394 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.6782 out_g_norm=0.5057 loss_all=0.6928 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=3850 epoch=3850/4000 epoch_step=1/1 micro_steps=3850 elapsed=3.2s lr=2.000000e-03 loss=0.6439 loss_recon=0.6439 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7372 corrupt_frac=1.0000 acc_corrupt=0.7372 loss_corrupt=0.6439 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6471 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8400 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9127 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9581 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.6851 out_g_norm=0.5525 loss_all=0.6515 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=3.2s lr=2.000000e-03 loss=0.6238 loss_recon=0.6238 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7406 corrupt_frac=1.0000 acc_corrupt=0.7406 loss_corrupt=0.6238 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6496 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8404 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.6917 out_g_norm=0.4961 loss_all=0.5879 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=3950 epoch=3950/4000 epoch_step=1/1 micro_steps=3950 elapsed=3.2s lr=2.000000e-03 loss=0.6289 loss_recon=0.6289 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7401 corrupt_frac=1.0000 acc_corrupt=0.7401 loss_corrupt=0.6289 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6464 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8392 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.6987 out_g_norm=0.4348 acc_corrupt_t_0p6_0p8=0.9607 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5629 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=3.2s lr=2.000000e-03 loss=0.6250 loss_recon=0.6250 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7425 corrupt_frac=1.0000 acc_corrupt=0.7425 loss_corrupt=0.6250 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6535 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8388 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9145 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7042 out_g_norm=0.4259 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5360 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4050 epoch=4050/4500 epoch_step=1/1 micro_steps=4050 elapsed=3.8s lr=2.000000e-03 loss=0.6238 loss_recon=0.6238 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7403 corrupt_frac=1.0000 acc_corrupt=0.7403 loss_corrupt=0.6238 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6534 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8391 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7085 out_g_norm=0.4086 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5829 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=4100 epoch=4100/4500 epoch_step=1/1 micro_steps=4100 elapsed=3.0s lr=2.000000e-03 loss=0.6039 loss_recon=0.6039 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7470 corrupt_frac=1.0000 acc_corrupt=0.7470 loss_corrupt=0.6039 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6587 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8405 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9138 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7104 out_g_norm=0.4962 acc_corrupt_t_0p6_0p8=0.9600 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7105 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=4150 epoch=4150/4500 epoch_step=1/1 micro_steps=4150 elapsed=3.0s lr=2.000000e-03 loss=0.6148 loss_recon=0.6148 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7427 corrupt_frac=1.0000 acc_corrupt=0.7427 loss_corrupt=0.6148 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6566 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7163 out_g_norm=0.4307 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7391 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=4200 epoch=4200/4500 epoch_step=1/1 micro_steps=4200 elapsed=3.0s lr=2.000000e-03 loss=0.6223 loss_recon=0.6223 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7426 corrupt_frac=1.0000 acc_corrupt=0.7426 loss_corrupt=0.6223 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6508 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7239 out_g_norm=0.4738 acc_corrupt_t_0p6_0p8=0.9585 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6450 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=4250 epoch=4250/4500 epoch_step=1/1 micro_steps=4250 elapsed=3.0s lr=2.000000e-03 loss=0.5892 loss_recon=0.5892 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7505 corrupt_frac=1.0000 acc_corrupt=0.7505 loss_corrupt=0.5892 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6639 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9141 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7241 out_g_norm=0.4952 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5946 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=4300 epoch=4300/4500 epoch_step=1/1 micro_steps=4300 elapsed=3.0s lr=2.000000e-03 loss=0.6037 loss_recon=0.6037 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7463 corrupt_frac=1.0000 acc_corrupt=0.7463 loss_corrupt=0.6037 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6553 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8400 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7352 out_g_norm=0.4417 loss_all=0.6464 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=4350 epoch=4350/4500 epoch_step=1/1 micro_steps=4350 elapsed=3.0s lr=2.000000e-03 loss=0.6145 loss_recon=0.6145 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7442 corrupt_frac=1.0000 acc_corrupt=0.7442 loss_corrupt=0.6145 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6596 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8399 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9128 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9584 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7328 out_g_norm=0.4506 loss_all=0.6247 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=4400 epoch=4400/4500 epoch_step=1/1 micro_steps=4400 elapsed=3.0s lr=2.000000e-03 loss=0.6060 loss_recon=0.6060 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7449 corrupt_frac=1.0000 acc_corrupt=0.7449 loss_corrupt=0.6060 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6574 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7307 out_g_norm=0.4537 loss_all=0.6310 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=4450 epoch=4450/4500 epoch_step=1/1 micro_steps=4450 elapsed=3.0s lr=2.000000e-03 loss=0.6000 loss_recon=0.6000 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7473 corrupt_frac=1.0000 acc_corrupt=0.7473 loss_corrupt=0.6000 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6587 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7366 out_g_norm=0.4084 acc_corrupt_t_0p6_0p8=0.9598 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5739 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=4500 epoch=4500/4500 epoch_step=1/1 micro_steps=4500 elapsed=3.0s lr=2.000000e-03 loss=0.6099 loss_recon=0.6099 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7454 corrupt_frac=1.0000 acc_corrupt=0.7454 loss_corrupt=0.6099 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6586 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8389 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9145 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7425 out_g_norm=0.4832 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5167 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4550 epoch=4550/5000 epoch_step=1/1 micro_steps=4550 elapsed=3.7s lr=2.000000e-03 loss=0.5948 loss_recon=0.5948 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7471 corrupt_frac=1.0000 acc_corrupt=0.7471 loss_corrupt=0.5948 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6656 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8385 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7477 out_g_norm=0.4230 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5272 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=3.0s lr=2.000000e-03 loss=0.5644 loss_recon=0.5644 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7583 corrupt_frac=1.0000 acc_corrupt=0.7583 loss_corrupt=0.5644 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6788 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9135 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7561 out_g_norm=0.4127 acc_corrupt_t_0p6_0p8=0.9609 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6626 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=4650 epoch=4650/5000 epoch_step=1/1 micro_steps=4650 elapsed=3.0s lr=2.000000e-03 loss=0.5950 loss_recon=0.5950 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7483 corrupt_frac=1.0000 acc_corrupt=0.7483 loss_corrupt=0.5950 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6668 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8398 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9630 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7614 out_g_norm=0.4340 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7172 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=3.0s lr=2.000000e-03 loss=0.5638 loss_recon=0.5638 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7567 corrupt_frac=1.0000 acc_corrupt=0.7567 loss_corrupt=0.5638 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6757 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7567 out_g_norm=0.3714 acc_corrupt_t_0p6_0p8=0.9579 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6097 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=4750 epoch=4750/5000 epoch_step=1/1 micro_steps=4750 elapsed=3.0s lr=2.000000e-03 loss=0.5603 loss_recon=0.5603 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7576 corrupt_frac=1.0000 acc_corrupt=0.7576 loss_corrupt=0.5603 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6764 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8409 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9639 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7741 out_g_norm=0.3978 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5989 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=3.0s lr=2.000000e-03 loss=0.5761 loss_recon=0.5761 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7546 corrupt_frac=1.0000 acc_corrupt=0.7546 loss_corrupt=0.5761 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6699 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8408 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7750 out_g_norm=0.3894 loss_all=0.6446 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=4850 epoch=4850/5000 epoch_step=1/1 micro_steps=4850 elapsed=3.0s lr=2.000000e-03 loss=0.5941 loss_recon=0.5941 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7493 corrupt_frac=1.0000 acc_corrupt=0.7493 loss_corrupt=0.5941 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6685 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9128 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9574 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7766 out_g_norm=0.4625 loss_all=0.6574 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=3.0s lr=2.000000e-03 loss=0.6019 loss_recon=0.6019 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7463 corrupt_frac=1.0000 acc_corrupt=0.7463 loss_corrupt=0.6019 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6598 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8403 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9144 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7643 out_g_norm=0.4464 loss_all=0.5252 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=4950 epoch=4950/5000 epoch_step=1/1 micro_steps=4950 elapsed=3.0s lr=2.000000e-03 loss=0.5785 loss_recon=0.5785 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7522 corrupt_frac=1.0000 acc_corrupt=0.7522 loss_corrupt=0.5785 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6677 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8401 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7664 out_g_norm=0.3747 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5390 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=3.0s lr=2.000000e-03 loss=0.5764 loss_recon=0.5764 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7535 corrupt_frac=1.0000 acc_corrupt=0.7535 loss_corrupt=0.5764 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6726 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8398 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7785 out_g_norm=0.3881 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5313 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5050 epoch=5050/5500 epoch_step=1/1 micro_steps=5050 elapsed=3.7s lr=2.000000e-03 loss=0.5659 loss_recon=0.5659 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7551 corrupt_frac=1.0000 acc_corrupt=0.7551 loss_corrupt=0.5659 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6791 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8396 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9636 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7832 out_g_norm=0.4015 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5408 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=5100 epoch=5100/5500 epoch_step=1/1 micro_steps=5100 elapsed=3.0s lr=2.000000e-03 loss=0.5561 loss_recon=0.5561 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7593 corrupt_frac=1.0000 acc_corrupt=0.7593 loss_corrupt=0.5561 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6802 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7907 out_g_norm=0.4493 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6852 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=5150 epoch=5150/5500 epoch_step=1/1 micro_steps=5150 elapsed=3.0s lr=2.000000e-03 loss=0.5726 loss_recon=0.5726 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7538 corrupt_frac=1.0000 acc_corrupt=0.7538 loss_corrupt=0.5726 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6759 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8407 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9145 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7944 out_g_norm=0.3787 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7252 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=5200 epoch=5200/5500 epoch_step=1/1 micro_steps=5200 elapsed=3.0s lr=2.000000e-03 loss=0.5755 loss_recon=0.5755 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7556 corrupt_frac=1.0000 acc_corrupt=0.7556 loss_corrupt=0.5755 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6739 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8408 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7891 out_g_norm=0.4302 acc_corrupt_t_0p6_0p8=0.9591 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6562 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=5250 epoch=5250/5500 epoch_step=1/1 micro_steps=5250 elapsed=3.0s lr=2.000000e-03 loss=0.5357 loss_recon=0.5357 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7646 corrupt_frac=1.0000 acc_corrupt=0.7646 loss_corrupt=0.5357 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6889 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9630 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7911 out_g_norm=0.3711 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5681 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=5300 epoch=5300/5500 epoch_step=1/1 micro_steps=5300 elapsed=3.0s lr=2.000000e-03 loss=0.5544 loss_recon=0.5544 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7593 corrupt_frac=1.0000 acc_corrupt=0.7593 loss_corrupt=0.5544 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6780 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8070 out_g_norm=0.3792 loss_all=0.5650 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=5350 epoch=5350/5500 epoch_step=1/1 micro_steps=5350 elapsed=3.0s lr=2.000000e-03 loss=0.5882 loss_recon=0.5882 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7495 corrupt_frac=1.0000 acc_corrupt=0.7495 loss_corrupt=0.5882 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6686 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8404 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9126 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9583 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8057 out_g_norm=0.3929 loss_all=0.5861 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=5400 epoch=5400/5500 epoch_step=1/1 micro_steps=5400 elapsed=3.0s lr=2.000000e-03 loss=0.5632 loss_recon=0.5632 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7570 corrupt_frac=1.0000 acc_corrupt=0.7570 loss_corrupt=0.5632 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6786 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8409 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7943 out_g_norm=0.3784 loss_all=0.5159 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=5450 epoch=5450/5500 epoch_step=1/1 micro_steps=5450 elapsed=3.0s lr=2.000000e-03 loss=0.5566 loss_recon=0.5566 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7589 corrupt_frac=1.0000 acc_corrupt=0.7589 loss_corrupt=0.5566 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6796 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8405 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7995 out_g_norm=0.3447 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6044 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=5500 epoch=5500/5500 epoch_step=1/1 micro_steps=5500 elapsed=3.0s lr=2.000000e-03 loss=0.5676 loss_recon=0.5676 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7562 corrupt_frac=1.0000 acc_corrupt=0.7562 loss_corrupt=0.5676 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6775 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8398 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8043 out_g_norm=0.4116 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5168 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5550 epoch=5550/6000 epoch_step=1/1 micro_steps=5550 elapsed=3.7s lr=2.000000e-03 loss=0.5596 loss_recon=0.5596 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7579 corrupt_frac=1.0000 acc_corrupt=0.7579 loss_corrupt=0.5596 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6839 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8398 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7991 out_g_norm=0.3971 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4912 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=3.0s lr=2.000000e-03 loss=0.5498 loss_recon=0.5498 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7617 corrupt_frac=1.0000 acc_corrupt=0.7617 loss_corrupt=0.5498 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6845 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7966 out_g_norm=0.4105 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6173 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=5650 epoch=5650/6000 epoch_step=1/1 micro_steps=5650 elapsed=3.0s lr=2.000000e-03 loss=0.5604 loss_recon=0.5604 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7578 corrupt_frac=1.0000 acc_corrupt=0.7578 loss_corrupt=0.5604 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6830 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8406 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8005 out_g_norm=0.3534 acc_corrupt_t_0p8_1p0=0.9876 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7020 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=3.0s lr=2.000000e-03 loss=0.5478 loss_recon=0.5478 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7608 corrupt_frac=1.0000 acc_corrupt=0.7608 loss_corrupt=0.5478 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6829 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8031 out_g_norm=0.3591 acc_corrupt_t_0p6_0p8=0.9594 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5589 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=5750 epoch=5750/6000 epoch_step=1/1 micro_steps=5750 elapsed=3.0s lr=2.000000e-03 loss=0.5286 loss_recon=0.5286 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7664 corrupt_frac=1.0000 acc_corrupt=0.7664 loss_corrupt=0.5286 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6919 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9138 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8206 out_g_norm=0.3146 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5616 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=3.0s lr=2.000000e-03 loss=0.5424 loss_recon=0.5424 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7620 corrupt_frac=1.0000 acc_corrupt=0.7620 loss_corrupt=0.5424 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6828 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8289 out_g_norm=0.3118 loss_all=0.6229 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=5850 epoch=5850/6000 epoch_step=1/1 micro_steps=5850 elapsed=3.0s lr=2.000000e-03 loss=0.5717 loss_recon=0.5717 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7541 corrupt_frac=1.0000 acc_corrupt=0.7541 loss_corrupt=0.5717 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6765 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8407 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9128 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9581 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8327 out_g_norm=0.3968 loss_all=0.4724 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=3.0s lr=2.000000e-03 loss=0.5605 loss_recon=0.5605 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7573 corrupt_frac=1.0000 acc_corrupt=0.7573 loss_corrupt=0.5605 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6791 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8408 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8242 out_g_norm=0.3516 loss_all=0.4564 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=5950 epoch=5950/6000 epoch_step=1/1 micro_steps=5950 elapsed=3.0s lr=2.000000e-03 loss=0.5370 loss_recon=0.5370 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7632 corrupt_frac=1.0000 acc_corrupt=0.7632 loss_corrupt=0.5370 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6870 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8332 out_g_norm=0.3458 acc_corrupt_t_0p6_0p8=0.9632 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6188 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=3.0s lr=2.000000e-03 loss=0.5596 loss_recon=0.5596 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7569 corrupt_frac=1.0000 acc_corrupt=0.7569 loss_corrupt=0.5596 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6782 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8405 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9632 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8372 out_g_norm=0.4003 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4784 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=6050 epoch=6050/6500 epoch_step=1/1 micro_steps=6050 elapsed=3.7s lr=2.000000e-03 loss=0.5633 loss_recon=0.5633 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7558 corrupt_frac=1.0000 acc_corrupt=0.7558 loss_corrupt=0.5633 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6804 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8395 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8319 out_g_norm=0.3794 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5244 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=6100 epoch=6100/6500 epoch_step=1/1 micro_steps=6100 elapsed=3.0s lr=2.000000e-03 loss=0.5312 loss_recon=0.5312 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7652 corrupt_frac=1.0000 acc_corrupt=0.7652 loss_corrupt=0.5312 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6908 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9144 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8342 out_g_norm=0.3290 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5802 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=6150 epoch=6150/6500 epoch_step=1/1 micro_steps=6150 elapsed=3.0s lr=2.000000e-03 loss=0.5586 loss_recon=0.5586 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7560 corrupt_frac=1.0000 acc_corrupt=0.7560 loss_corrupt=0.5586 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6797 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8408 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8352 out_g_norm=0.3657 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7376 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=6200 epoch=6200/6500 epoch_step=1/1 micro_steps=6200 elapsed=3.0s lr=2.000000e-03 loss=0.5523 loss_recon=0.5523 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7601 corrupt_frac=1.0000 acc_corrupt=0.7601 loss_corrupt=0.5523 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6818 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8295 out_g_norm=0.3989 acc_corrupt_t_0p6_0p8=0.9590 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6422 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=6250 epoch=6250/6500 epoch_step=1/1 micro_steps=6250 elapsed=3.0s lr=2.000000e-03 loss=0.5181 loss_recon=0.5181 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7686 corrupt_frac=1.0000 acc_corrupt=0.7686 loss_corrupt=0.5181 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6961 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9145 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9644 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8391 out_g_norm=0.3362 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5893 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=6300 epoch=6300/6500 epoch_step=1/1 micro_steps=6300 elapsed=3.0s lr=2.000000e-03 loss=0.5512 loss_recon=0.5512 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7611 corrupt_frac=1.0000 acc_corrupt=0.7611 loss_corrupt=0.5512 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6816 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8416 out_g_norm=0.3717 loss_all=0.5701 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=6350 epoch=6350/6500 epoch_step=1/1 micro_steps=6350 elapsed=3.0s lr=2.000000e-03 loss=0.5483 loss_recon=0.5483 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7576 corrupt_frac=1.0000 acc_corrupt=0.7576 loss_corrupt=0.5483 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6822 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9136 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9573 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8442 out_g_norm=0.3158 loss_all=0.4907 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=6400 epoch=6400/6500 epoch_step=1/1 micro_steps=6400 elapsed=3.0s lr=2.000000e-03 loss=0.5353 loss_recon=0.5353 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7625 corrupt_frac=1.0000 acc_corrupt=0.7625 loss_corrupt=0.5353 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6884 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8409 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9148 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8474 out_g_norm=0.3377 loss_all=0.4881 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=6450 epoch=6450/6500 epoch_step=1/1 micro_steps=6450 elapsed=3.0s lr=2.000000e-03 loss=0.5418 loss_recon=0.5418 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7626 corrupt_frac=1.0000 acc_corrupt=0.7626 loss_corrupt=0.5418 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6865 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8403 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8463 out_g_norm=0.3867 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5054 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=6500 epoch=6500/6500 epoch_step=1/1 micro_steps=6500 elapsed=3.0s lr=2.000000e-03 loss=0.5395 loss_recon=0.5395 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7616 corrupt_frac=1.0000 acc_corrupt=0.7616 loss_corrupt=0.5395 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6864 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8368 out_g_norm=0.3420 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5503 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=6501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=6550 epoch=6550/7000 epoch_step=1/1 micro_steps=6550 elapsed=4.0s lr=2.000000e-03 loss=0.5487 loss_recon=0.5487 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7594 corrupt_frac=1.0000 acc_corrupt=0.7594 loss_corrupt=0.5487 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6865 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8399 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8361 out_g_norm=0.3522 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4993 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=6600 epoch=6600/7000 epoch_step=1/1 micro_steps=6600 elapsed=3.1s lr=2.000000e-03 loss=0.5270 loss_recon=0.5270 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7658 corrupt_frac=1.0000 acc_corrupt=0.7658 loss_corrupt=0.5270 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6916 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9145 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8442 out_g_norm=0.3091 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5860 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=6650 epoch=6650/7000 epoch_step=1/1 micro_steps=6650 elapsed=3.1s lr=2.000000e-03 loss=0.5313 loss_recon=0.5313 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7625 corrupt_frac=1.0000 acc_corrupt=0.7625 loss_corrupt=0.5313 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6909 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8433 out_g_norm=0.3192 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6494 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=6700 epoch=6700/7000 epoch_step=1/1 micro_steps=6700 elapsed=3.1s lr=2.000000e-03 loss=0.5372 loss_recon=0.5372 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7634 corrupt_frac=1.0000 acc_corrupt=0.7634 loss_corrupt=0.5372 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6880 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8484 out_g_norm=0.3632 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6658 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=6750 epoch=6750/7000 epoch_step=1/1 micro_steps=6750 elapsed=3.1s lr=2.000000e-03 loss=0.5278 loss_recon=0.5278 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7658 corrupt_frac=1.0000 acc_corrupt=0.7658 loss_corrupt=0.5278 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6912 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8469 out_g_norm=0.3368 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5790 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=6800 epoch=6800/7000 epoch_step=1/1 micro_steps=6800 elapsed=3.1s lr=2.000000e-03 loss=0.5240 loss_recon=0.5240 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7670 corrupt_frac=1.0000 acc_corrupt=0.7670 loss_corrupt=0.5240 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6919 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9166 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8498 out_g_norm=0.3009 loss_all=0.4819 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=6850 epoch=6850/7000 epoch_step=1/1 micro_steps=6850 elapsed=3.1s lr=2.000000e-03 loss=0.5376 loss_recon=0.5376 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7616 corrupt_frac=1.0000 acc_corrupt=0.7616 loss_corrupt=0.5376 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6894 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9135 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9587 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8540 out_g_norm=0.3304 loss_all=0.5396 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=6900 epoch=6900/7000 epoch_step=1/1 micro_steps=6900 elapsed=3.1s lr=2.000000e-03 loss=0.5356 loss_recon=0.5356 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7634 corrupt_frac=1.0000 acc_corrupt=0.7634 loss_corrupt=0.5356 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6900 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9148 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9632 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8519 out_g_norm=0.3418 loss_all=0.4941 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=6950 epoch=6950/7000 epoch_step=1/1 micro_steps=6950 elapsed=3.1s lr=2.000000e-03 loss=0.5417 loss_recon=0.5417 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7625 corrupt_frac=1.0000 acc_corrupt=0.7625 loss_corrupt=0.5417 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6858 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8407 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8507 out_g_norm=0.3807 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5611 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=7000 epoch=7000/7000 epoch_step=1/1 micro_steps=7000 elapsed=3.1s lr=2.000000e-03 loss=0.5308 loss_recon=0.5308 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7644 corrupt_frac=1.0000 acc_corrupt=0.7644 loss_corrupt=0.5308 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6913 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8471 out_g_norm=0.3377 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5781 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=7001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=7050 epoch=7050/7500 epoch_step=1/1 micro_steps=7050 elapsed=3.9s lr=2.000000e-03 loss=0.5315 loss_recon=0.5315 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7628 corrupt_frac=1.0000 acc_corrupt=0.7628 loss_corrupt=0.5315 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6920 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8405 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8420 out_g_norm=0.3012 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5108 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=7100 epoch=7100/7500 epoch_step=1/1 micro_steps=7100 elapsed=3.1s lr=2.000000e-03 loss=0.5143 loss_recon=0.5143 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7684 corrupt_frac=1.0000 acc_corrupt=0.7684 loss_corrupt=0.5143 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6962 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8537 out_g_norm=0.3019 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5081 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=7150 epoch=7150/7500 epoch_step=1/1 micro_steps=7150 elapsed=3.1s lr=2.000000e-03 loss=0.5250 loss_recon=0.5250 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7649 corrupt_frac=1.0000 acc_corrupt=0.7649 loss_corrupt=0.5250 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6952 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8536 out_g_norm=0.3699 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5955 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=7200 epoch=7200/7500 epoch_step=1/1 micro_steps=7200 elapsed=3.1s lr=2.000000e-03 loss=0.5163 loss_recon=0.5163 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7681 corrupt_frac=1.0000 acc_corrupt=0.7681 loss_corrupt=0.5163 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6956 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8534 out_g_norm=0.3176 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6322 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=7250 epoch=7250/7500 epoch_step=1/1 micro_steps=7250 elapsed=3.1s lr=2.000000e-03 loss=0.5102 loss_recon=0.5102 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7703 corrupt_frac=1.0000 acc_corrupt=0.7703 loss_corrupt=0.5102 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6989 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9640 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8562 out_g_norm=0.3116 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5983 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=7300 epoch=7300/7500 epoch_step=1/1 micro_steps=7300 elapsed=3.1s lr=2.000000e-03 loss=0.5345 loss_recon=0.5345 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7637 corrupt_frac=1.0000 acc_corrupt=0.7637 loss_corrupt=0.5345 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6860 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9174 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8644 out_g_norm=0.3544 loss_all=0.5848 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=7350 epoch=7350/7500 epoch_step=1/1 micro_steps=7350 elapsed=3.1s lr=2.000000e-03 loss=0.5333 loss_recon=0.5333 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7628 corrupt_frac=1.0000 acc_corrupt=0.7628 loss_corrupt=0.5333 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6915 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9132 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9588 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8574 out_g_norm=0.3306 loss_all=0.5117 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=7400 epoch=7400/7500 epoch_step=1/1 micro_steps=7400 elapsed=3.1s lr=2.000000e-03 loss=0.5410 loss_recon=0.5410 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7608 corrupt_frac=1.0000 acc_corrupt=0.7608 loss_corrupt=0.5410 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6853 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8409 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8526 out_g_norm=0.3632 loss_all=0.4875 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=7450 epoch=7450/7500 epoch_step=1/1 micro_steps=7450 elapsed=3.1s lr=2.000000e-03 loss=0.5339 loss_recon=0.5339 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7619 corrupt_frac=1.0000 acc_corrupt=0.7619 loss_corrupt=0.5339 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6846 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8483 out_g_norm=0.3378 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5281 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=7500 epoch=7500/7500 epoch_step=1/1 micro_steps=7500 elapsed=3.1s lr=2.000000e-03 loss=0.5449 loss_recon=0.5449 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7598 corrupt_frac=1.0000 acc_corrupt=0.7598 loss_corrupt=0.5449 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6836 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8402 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9147 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8537 out_g_norm=0.3696 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5554 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=7501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=7550 epoch=7550/8000 epoch_step=1/1 micro_steps=7550 elapsed=3.7s lr=2.000000e-03 loss=0.5337 loss_recon=0.5337 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7619 corrupt_frac=1.0000 acc_corrupt=0.7619 loss_corrupt=0.5337 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6908 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8399 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8534 out_g_norm=0.3925 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5348 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=7600 epoch=7600/8000 epoch_step=1/1 micro_steps=7600 elapsed=3.0s lr=2.000000e-03 loss=0.5428 loss_recon=0.5428 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7613 corrupt_frac=1.0000 acc_corrupt=0.7613 loss_corrupt=0.5428 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6839 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8537 out_g_norm=0.4181 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5404 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=7650 epoch=7650/8000 epoch_step=1/1 micro_steps=7650 elapsed=3.0s lr=2.000000e-03 loss=0.5215 loss_recon=0.5215 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7646 corrupt_frac=1.0000 acc_corrupt=0.7646 loss_corrupt=0.5215 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6946 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8544 out_g_norm=0.3198 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6640 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=7700 epoch=7700/8000 epoch_step=1/1 micro_steps=7700 elapsed=3.0s lr=2.000000e-03 loss=0.5234 loss_recon=0.5234 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7652 corrupt_frac=1.0000 acc_corrupt=0.7652 loss_corrupt=0.5234 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6906 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8562 out_g_norm=0.3150 acc_corrupt_t_0p6_0p8=0.9589 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6047 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=7750 epoch=7750/8000 epoch_step=1/1 micro_steps=7750 elapsed=3.0s lr=2.000000e-03 loss=0.5102 loss_recon=0.5102 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7696 corrupt_frac=1.0000 acc_corrupt=0.7696 loss_corrupt=0.5102 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6981 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8579 out_g_norm=0.3356 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5295 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=7800 epoch=7800/8000 epoch_step=1/1 micro_steps=7800 elapsed=3.0s lr=2.000000e-03 loss=0.5312 loss_recon=0.5312 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7650 corrupt_frac=1.0000 acc_corrupt=0.7650 loss_corrupt=0.5312 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6883 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8692 out_g_norm=0.3638 loss_all=0.5791 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=7850 epoch=7850/8000 epoch_step=1/1 micro_steps=7850 elapsed=3.0s lr=2.000000e-03 loss=0.5289 loss_recon=0.5289 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7636 corrupt_frac=1.0000 acc_corrupt=0.7636 loss_corrupt=0.5289 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6930 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9137 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9590 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8690 out_g_norm=0.3036 loss_all=0.4707 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=7900 epoch=7900/8000 epoch_step=1/1 micro_steps=7900 elapsed=3.0s lr=2.000000e-03 loss=0.5180 loss_recon=0.5180 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7671 corrupt_frac=1.0000 acc_corrupt=0.7671 loss_corrupt=0.5180 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6963 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8794 out_g_norm=0.2971 loss_all=0.4796 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=7950 epoch=7950/8000 epoch_step=1/1 micro_steps=7950 elapsed=3.0s lr=2.000000e-03 loss=0.5362 loss_recon=0.5362 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7628 corrupt_frac=1.0000 acc_corrupt=0.7628 loss_corrupt=0.5362 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6865 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8406 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8859 out_g_norm=0.3112 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5939 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=8000 epoch=8000/8000 epoch_step=1/1 micro_steps=8000 elapsed=3.0s lr=2.000000e-03 loss=0.5311 loss_recon=0.5311 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7643 corrupt_frac=1.0000 acc_corrupt=0.7643 loss_corrupt=0.5311 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6914 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8407 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8768 out_g_norm=0.3478 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5754 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=8001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=8050 epoch=8050/8500 epoch_step=1/1 micro_steps=8050 elapsed=3.7s lr=2.000000e-03 loss=0.5064 loss_recon=0.5064 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7689 corrupt_frac=1.0000 acc_corrupt=0.7689 loss_corrupt=0.5064 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7029 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8403 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9640 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8811 out_g_norm=0.2743 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5250 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=8100 epoch=8100/8500 epoch_step=1/1 micro_steps=8100 elapsed=3.0s lr=2.000000e-03 loss=0.5122 loss_recon=0.5122 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7688 corrupt_frac=1.0000 acc_corrupt=0.7688 loss_corrupt=0.5122 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6969 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8937 out_g_norm=0.2922 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5079 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=8150 epoch=8150/8500 epoch_step=1/1 micro_steps=8150 elapsed=3.0s lr=2.000000e-03 loss=0.5152 loss_recon=0.5152 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7669 corrupt_frac=1.0000 acc_corrupt=0.7669 loss_corrupt=0.5152 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6987 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8961 out_g_norm=0.2988 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6497 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=8200 epoch=8200/8500 epoch_step=1/1 micro_steps=8200 elapsed=3.0s lr=2.000000e-03 loss=0.5120 loss_recon=0.5120 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7697 corrupt_frac=1.0000 acc_corrupt=0.7697 loss_corrupt=0.5120 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6989 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8867 out_g_norm=0.3198 acc_corrupt_t_0p6_0p8=0.9607 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5710 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=8250 epoch=8250/8500 epoch_step=1/1 micro_steps=8250 elapsed=3.0s lr=2.000000e-03 loss=0.4979 loss_recon=0.4979 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7725 corrupt_frac=1.0000 acc_corrupt=0.7725 loss_corrupt=0.4979 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7028 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9641 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8852 out_g_norm=0.3407 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4938 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=8300 epoch=8300/8500 epoch_step=1/1 micro_steps=8300 elapsed=3.0s lr=2.000000e-03 loss=0.5123 loss_recon=0.5123 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7694 corrupt_frac=1.0000 acc_corrupt=0.7694 loss_corrupt=0.5123 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6961 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9174 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8894 out_g_norm=0.3411 loss_all=0.6114 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=8350 epoch=8350/8500 epoch_step=1/1 micro_steps=8350 elapsed=3.0s lr=2.000000e-03 loss=0.5167 loss_recon=0.5167 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7661 corrupt_frac=1.0000 acc_corrupt=0.7661 loss_corrupt=0.5167 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6970 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9148 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9586 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8890 out_g_norm=0.3116 loss_all=0.5131 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=8400 epoch=8400/8500 epoch_step=1/1 micro_steps=8400 elapsed=3.0s lr=2.000000e-03 loss=0.5421 loss_recon=0.5421 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7611 corrupt_frac=1.0000 acc_corrupt=0.7611 loss_corrupt=0.5421 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6858 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8875 out_g_norm=0.3478 loss_all=0.5473 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=8450 epoch=8450/8500 epoch_step=1/1 micro_steps=8450 elapsed=3.1s lr=2.000000e-03 loss=0.5350 loss_recon=0.5350 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7626 corrupt_frac=1.0000 acc_corrupt=0.7626 loss_corrupt=0.5350 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6863 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8404 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8823 out_g_norm=0.3214 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6313 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=8500 epoch=8500/8500 epoch_step=1/1 micro_steps=8500 elapsed=3.1s lr=2.000000e-03 loss=0.5200 loss_recon=0.5200 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7667 corrupt_frac=1.0000 acc_corrupt=0.7667 loss_corrupt=0.5200 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6955 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8769 out_g_norm=0.3135 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5411 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=8501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=8550 epoch=8550/9000 epoch_step=1/1 micro_steps=8550 elapsed=3.8s lr=2.000000e-03 loss=0.4969 loss_recon=0.4969 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7708 corrupt_frac=1.0000 acc_corrupt=0.7708 loss_corrupt=0.4969 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7065 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8401 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8841 out_g_norm=0.2936 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5554 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=8600 epoch=8600/9000 epoch_step=1/1 micro_steps=8600 elapsed=3.1s lr=2.000000e-03 loss=0.4926 loss_recon=0.4926 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7726 corrupt_frac=1.0000 acc_corrupt=0.7726 loss_corrupt=0.4926 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7034 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8918 out_g_norm=0.2748 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5189 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=8650 epoch=8650/9000 epoch_step=1/1 micro_steps=8650 elapsed=3.1s lr=2.000000e-03 loss=0.5230 loss_recon=0.5230 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7642 corrupt_frac=1.0000 acc_corrupt=0.7642 loss_corrupt=0.5230 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6936 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8890 out_g_norm=0.3138 acc_corrupt_t_0p8_1p0=0.9889 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6074 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=8700 epoch=8700/9000 epoch_step=1/1 micro_steps=8700 elapsed=3.1s lr=2.000000e-03 loss=0.5106 loss_recon=0.5106 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7687 corrupt_frac=1.0000 acc_corrupt=0.7687 loss_corrupt=0.5106 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6966 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8842 out_g_norm=0.3016 acc_corrupt_t_0p6_0p8=0.9603 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5333 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=8750 epoch=8750/9000 epoch_step=1/1 micro_steps=8750 elapsed=3.1s lr=2.000000e-03 loss=0.5083 loss_recon=0.5083 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7704 corrupt_frac=1.0000 acc_corrupt=0.7704 loss_corrupt=0.5083 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6988 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9642 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8856 out_g_norm=0.3228 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4775 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=8800 epoch=8800/9000 epoch_step=1/1 micro_steps=8800 elapsed=3.1s lr=2.000000e-03 loss=0.5021 loss_recon=0.5021 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7720 corrupt_frac=1.0000 acc_corrupt=0.7720 loss_corrupt=0.5021 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7009 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8882 out_g_norm=0.2844 loss_all=0.6147 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=8850 epoch=8850/9000 epoch_step=1/1 micro_steps=8850 elapsed=3.1s lr=2.000000e-03 loss=0.5131 loss_recon=0.5131 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7682 corrupt_frac=1.0000 acc_corrupt=0.7682 loss_corrupt=0.5131 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7008 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9138 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9588 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8886 out_g_norm=0.3168 loss_all=0.5007 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=8900 epoch=8900/9000 epoch_step=1/1 micro_steps=8900 elapsed=3.1s lr=2.000000e-03 loss=0.5074 loss_recon=0.5074 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7687 corrupt_frac=1.0000 acc_corrupt=0.7687 loss_corrupt=0.5074 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6988 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8879 out_g_norm=0.2744 loss_all=0.4496 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=8950 epoch=8950/9000 epoch_step=1/1 micro_steps=8950 elapsed=3.2s lr=2.000000e-03 loss=0.5049 loss_recon=0.5049 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7695 corrupt_frac=1.0000 acc_corrupt=0.7695 loss_corrupt=0.5049 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6983 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8940 out_g_norm=0.2873 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4950 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=9000 epoch=9000/9000 epoch_step=1/1 micro_steps=9000 elapsed=3.2s lr=2.000000e-03 loss=0.5159 loss_recon=0.5159 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7661 corrupt_frac=1.0000 acc_corrupt=0.7661 loss_corrupt=0.5159 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.6943 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8991 out_g_norm=0.2855 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4853 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=9001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=9050 epoch=9050/9500 epoch_step=1/1 micro_steps=9050 elapsed=3.7s lr=2.000000e-03 loss=0.5101 loss_recon=0.5101 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7672 corrupt_frac=1.0000 acc_corrupt=0.7672 loss_corrupt=0.5101 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7000 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8403 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8930 out_g_norm=0.2725 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5316 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=9100 epoch=9100/9500 epoch_step=1/1 micro_steps=9100 elapsed=3.0s lr=2.000000e-03 loss=0.4862 loss_recon=0.4862 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7747 corrupt_frac=1.0000 acc_corrupt=0.7747 loss_corrupt=0.4862 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7072 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8969 out_g_norm=0.2696 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5582 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=9150 epoch=9150/9500 epoch_step=1/1 micro_steps=9150 elapsed=3.2s lr=2.000000e-03 loss=0.5313 loss_recon=0.5313 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7623 corrupt_frac=1.0000 acc_corrupt=0.7623 loss_corrupt=0.5313 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6904 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8970 out_g_norm=0.2821 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6173 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=9200 epoch=9200/9500 epoch_step=1/1 micro_steps=9200 elapsed=3.1s lr=2.000000e-03 loss=0.5090 loss_recon=0.5090 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7701 corrupt_frac=1.0000 acc_corrupt=0.7701 loss_corrupt=0.5090 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.6994 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8868 out_g_norm=0.2719 acc_corrupt_t_0p6_0p8=0.9600 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5774 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=9250 epoch=9250/9500 epoch_step=1/1 micro_steps=9250 elapsed=3.2s lr=2.000000e-03 loss=0.5066 loss_recon=0.5066 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7697 corrupt_frac=1.0000 acc_corrupt=0.7697 loss_corrupt=0.5066 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6981 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9641 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8865 out_g_norm=0.3234 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5445 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=9300 epoch=9300/9500 epoch_step=1/1 micro_steps=9300 elapsed=3.1s lr=2.000000e-03 loss=0.5058 loss_recon=0.5058 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7707 corrupt_frac=1.0000 acc_corrupt=0.7707 loss_corrupt=0.5058 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6983 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9168 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8881 out_g_norm=0.2787 loss_all=0.5910 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=9350 epoch=9350/9500 epoch_step=1/1 micro_steps=9350 elapsed=3.1s lr=2.000000e-03 loss=0.5192 loss_recon=0.5192 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7649 corrupt_frac=1.0000 acc_corrupt=0.7649 loss_corrupt=0.5192 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.6952 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9138 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9587 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8871 out_g_norm=0.3077 loss_all=0.4666 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=9400 epoch=9400/9500 epoch_step=1/1 micro_steps=9400 elapsed=3.1s lr=2.000000e-03 loss=0.5020 loss_recon=0.5020 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7696 corrupt_frac=1.0000 acc_corrupt=0.7696 loss_corrupt=0.5020 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7003 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8888 out_g_norm=0.2790 loss_all=0.5033 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=9450 epoch=9450/9500 epoch_step=1/1 micro_steps=9450 elapsed=3.1s lr=2.000000e-03 loss=0.5165 loss_recon=0.5165 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7674 corrupt_frac=1.0000 acc_corrupt=0.7674 loss_corrupt=0.5165 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6942 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8880 out_g_norm=0.2912 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5739 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=9500 epoch=9500/9500 epoch_step=1/1 micro_steps=9500 elapsed=3.1s lr=2.000000e-03 loss=0.4915 loss_recon=0.4915 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7731 corrupt_frac=1.0000 acc_corrupt=0.7731 loss_corrupt=0.4915 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7066 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9636 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8887 out_g_norm=0.2650 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5019 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=9501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=9550 epoch=9550/10000 epoch_step=1/1 micro_steps=9550 elapsed=3.9s lr=2.000000e-03 loss=0.5122 loss_recon=0.5122 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7665 corrupt_frac=1.0000 acc_corrupt=0.7665 loss_corrupt=0.5122 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.6984 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8408 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9638 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8931 out_g_norm=0.2863 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4336 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=9600 epoch=9600/10000 epoch_step=1/1 micro_steps=9600 elapsed=3.1s lr=2.000000e-03 loss=0.4982 loss_recon=0.4982 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7709 corrupt_frac=1.0000 acc_corrupt=0.7709 loss_corrupt=0.4982 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7007 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8943 out_g_norm=0.3129 acc_corrupt_t_0p6_0p8=0.9611 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5963 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=9650 epoch=9650/10000 epoch_step=1/1 micro_steps=9650 elapsed=3.1s lr=2.000000e-03 loss=0.5104 loss_recon=0.5104 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7681 corrupt_frac=1.0000 acc_corrupt=0.7681 loss_corrupt=0.5104 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7005 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8890 out_g_norm=0.3001 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6427 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=9700 epoch=9700/10000 epoch_step=1/1 micro_steps=9700 elapsed=3.1s lr=2.000000e-03 loss=0.5013 loss_recon=0.5013 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7710 corrupt_frac=1.0000 acc_corrupt=0.7710 loss_corrupt=0.5013 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7010 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8894 out_g_norm=0.2823 acc_corrupt_t_0p6_0p8=0.9609 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4703 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=9750 epoch=9750/10000 epoch_step=1/1 micro_steps=9750 elapsed=3.2s lr=2.000000e-03 loss=0.4977 loss_recon=0.4977 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7718 corrupt_frac=1.0000 acc_corrupt=0.7718 loss_corrupt=0.4977 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7019 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9646 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8853 out_g_norm=0.3435 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5632 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=9800 epoch=9800/10000 epoch_step=1/1 micro_steps=9800 elapsed=3.2s lr=2.000000e-03 loss=0.5061 loss_recon=0.5061 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7713 corrupt_frac=1.0000 acc_corrupt=0.7713 loss_corrupt=0.5061 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6996 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8911 out_g_norm=0.3128 loss_all=0.5566 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=9850 epoch=9850/10000 epoch_step=1/1 micro_steps=9850 elapsed=3.2s lr=2.000000e-03 loss=0.5085 loss_recon=0.5085 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7682 corrupt_frac=1.0000 acc_corrupt=0.7682 loss_corrupt=0.5085 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7008 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9142 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9580 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8863 out_g_norm=0.3011 loss_all=0.4451 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=9900 epoch=9900/10000 epoch_step=1/1 micro_steps=9900 elapsed=3.1s lr=2.000000e-03 loss=0.5054 loss_recon=0.5054 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7707 corrupt_frac=1.0000 acc_corrupt=0.7707 loss_corrupt=0.5054 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7023 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8840 out_g_norm=0.2784 loss_all=0.5065 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=9950 epoch=9950/10000 epoch_step=1/1 micro_steps=9950 elapsed=3.1s lr=2.000000e-03 loss=0.5186 loss_recon=0.5186 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7664 corrupt_frac=1.0000 acc_corrupt=0.7664 loss_corrupt=0.5186 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6926 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8783 out_g_norm=0.3165 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5876 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=10000 epoch=10000/10000 epoch_step=1/1 micro_steps=10000 elapsed=3.1s lr=2.000000e-03 loss=0.4991 loss_recon=0.4991 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7718 corrupt_frac=1.0000 acc_corrupt=0.7718 loss_corrupt=0.4991 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7046 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8664 out_g_norm=0.2909 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4581 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=10001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=10050 epoch=10050/10500 epoch_step=1/1 micro_steps=10050 elapsed=3.7s lr=2.000000e-03 loss=0.5059 loss_recon=0.5059 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7676 corrupt_frac=1.0000 acc_corrupt=0.7676 loss_corrupt=0.5059 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7005 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8407 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8712 out_g_norm=0.2648 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5185 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=10100 epoch=10100/10500 epoch_step=1/1 micro_steps=10100 elapsed=3.0s lr=2.000000e-03 loss=0.4979 loss_recon=0.4979 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7719 corrupt_frac=1.0000 acc_corrupt=0.7719 loss_corrupt=0.4979 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7024 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8698 out_g_norm=0.3101 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5483 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=10150 epoch=10150/10500 epoch_step=1/1 micro_steps=10150 elapsed=3.0s lr=2.000000e-03 loss=0.5062 loss_recon=0.5062 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7691 corrupt_frac=1.0000 acc_corrupt=0.7691 loss_corrupt=0.5062 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7021 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8663 out_g_norm=0.2807 acc_corrupt_t_0p8_1p0=0.9876 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5902 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=10200 epoch=10200/10500 epoch_step=1/1 micro_steps=10200 elapsed=3.0s lr=2.000000e-03 loss=0.4882 loss_recon=0.4882 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7741 corrupt_frac=1.0000 acc_corrupt=0.7741 loss_corrupt=0.4882 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7066 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8761 out_g_norm=0.2596 acc_corrupt_t_0p6_0p8=0.9603 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5227 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=10250 epoch=10250/10500 epoch_step=1/1 micro_steps=10250 elapsed=3.0s lr=2.000000e-03 loss=0.5100 loss_recon=0.5100 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7689 corrupt_frac=1.0000 acc_corrupt=0.7689 loss_corrupt=0.5100 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.6962 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9642 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8740 out_g_norm=0.3100 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5153 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=10300 epoch=10300/10500 epoch_step=1/1 micro_steps=10300 elapsed=3.0s lr=2.000000e-03 loss=0.5013 loss_recon=0.5013 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7720 corrupt_frac=1.0000 acc_corrupt=0.7720 loss_corrupt=0.5013 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7010 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9172 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8694 out_g_norm=0.3107 loss_all=0.4969 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=10350 epoch=10350/10500 epoch_step=1/1 micro_steps=10350 elapsed=3.0s lr=2.000000e-03 loss=0.4971 loss_recon=0.4971 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7710 corrupt_frac=1.0000 acc_corrupt=0.7710 loss_corrupt=0.4971 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7054 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9596 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8732 out_g_norm=0.2591 loss_all=0.5067 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=10400 epoch=10400/10500 epoch_step=1/1 micro_steps=10400 elapsed=3.0s lr=2.000000e-03 loss=0.5186 loss_recon=0.5186 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7666 corrupt_frac=1.0000 acc_corrupt=0.7666 loss_corrupt=0.5186 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.6950 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8727 out_g_norm=0.3028 loss_all=0.4823 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=10450 epoch=10450/10500 epoch_step=1/1 micro_steps=10450 elapsed=3.0s lr=2.000000e-03 loss=0.4962 loss_recon=0.4962 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7720 corrupt_frac=1.0000 acc_corrupt=0.7720 loss_corrupt=0.4962 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7025 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8685 out_g_norm=0.3098 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5094 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=10500 epoch=10500/10500 epoch_step=1/1 micro_steps=10500 elapsed=3.0s lr=2.000000e-03 loss=0.4944 loss_recon=0.4944 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7722 corrupt_frac=1.0000 acc_corrupt=0.7722 loss_corrupt=0.4944 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7050 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8681 out_g_norm=0.2980 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4863 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=10501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=10550 epoch=10550/11000 epoch_step=1/1 micro_steps=10550 elapsed=3.7s lr=2.000000e-03 loss=0.4970 loss_recon=0.4970 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7714 corrupt_frac=1.0000 acc_corrupt=0.7714 loss_corrupt=0.4970 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7069 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8771 out_g_norm=0.2683 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5642 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=10600 epoch=10600/11000 epoch_step=1/1 micro_steps=10600 elapsed=3.0s lr=2.000000e-03 loss=0.4914 loss_recon=0.4914 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7732 corrupt_frac=1.0000 acc_corrupt=0.7732 loss_corrupt=0.4914 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7047 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9149 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8766 out_g_norm=0.2972 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5662 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=10650 epoch=10650/11000 epoch_step=1/1 micro_steps=10650 elapsed=3.0s lr=2.000000e-03 loss=0.4997 loss_recon=0.4997 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7692 corrupt_frac=1.0000 acc_corrupt=0.7692 loss_corrupt=0.4997 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7024 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8805 out_g_norm=0.2931 acc_corrupt_t_0p8_1p0=0.9909 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6029 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=10700 epoch=10700/11000 epoch_step=1/1 micro_steps=10700 elapsed=3.0s lr=2.000000e-03 loss=0.4880 loss_recon=0.4880 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7733 corrupt_frac=1.0000 acc_corrupt=0.7733 loss_corrupt=0.4880 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7053 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8843 out_g_norm=0.3068 acc_corrupt_t_0p6_0p8=0.9593 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4964 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=10750 epoch=10750/11000 epoch_step=1/1 micro_steps=10750 elapsed=3.0s lr=2.000000e-03 loss=0.4898 loss_recon=0.4898 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4898 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7047 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9641 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8871 out_g_norm=0.3008 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4837 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=10800 epoch=10800/11000 epoch_step=1/1 micro_steps=10800 elapsed=3.0s lr=2.000000e-03 loss=0.5083 loss_recon=0.5083 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7703 corrupt_frac=1.0000 acc_corrupt=0.7703 loss_corrupt=0.5083 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.6979 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8933 out_g_norm=0.3187 loss_all=0.4978 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=10850 epoch=10850/11000 epoch_step=1/1 micro_steps=10850 elapsed=3.0s lr=2.000000e-03 loss=0.5002 loss_recon=0.5002 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7701 corrupt_frac=1.0000 acc_corrupt=0.7701 loss_corrupt=0.5002 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7041 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9591 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8880 out_g_norm=0.2963 loss_all=0.4830 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=10900 epoch=10900/11000 epoch_step=1/1 micro_steps=10900 elapsed=3.0s lr=2.000000e-03 loss=0.4799 loss_recon=0.4799 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7765 corrupt_frac=1.0000 acc_corrupt=0.7765 loss_corrupt=0.4799 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7122 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8918 out_g_norm=0.2651 loss_all=0.4118 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=10950 epoch=10950/11000 epoch_step=1/1 micro_steps=10950 elapsed=3.0s lr=2.000000e-03 loss=0.4945 loss_recon=0.4945 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7716 corrupt_frac=1.0000 acc_corrupt=0.7716 loss_corrupt=0.4945 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7018 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9164 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8956 out_g_norm=0.2702 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5220 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=11000 epoch=11000/11000 epoch_step=1/1 micro_steps=11000 elapsed=3.0s lr=2.000000e-03 loss=0.4998 loss_recon=0.4998 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7715 corrupt_frac=1.0000 acc_corrupt=0.7715 loss_corrupt=0.4998 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7038 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8870 out_g_norm=0.3269 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5235 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=11001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=11050 epoch=11050/11500 epoch_step=1/1 micro_steps=11050 elapsed=3.7s lr=2.000000e-03 loss=0.4953 loss_recon=0.4953 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7709 corrupt_frac=1.0000 acc_corrupt=0.7709 loss_corrupt=0.4953 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7067 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8401 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8847 out_g_norm=0.2750 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5171 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=11100 epoch=11100/11500 epoch_step=1/1 micro_steps=11100 elapsed=3.0s lr=2.000000e-03 loss=0.5017 loss_recon=0.5017 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7702 corrupt_frac=1.0000 acc_corrupt=0.7702 loss_corrupt=0.5017 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.6995 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9144 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8836 out_g_norm=0.3193 acc_corrupt_t_0p6_0p8=0.9604 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5850 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=11150 epoch=11150/11500 epoch_step=1/1 micro_steps=11150 elapsed=3.0s lr=2.000000e-03 loss=0.5124 loss_recon=0.5124 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7670 corrupt_frac=1.0000 acc_corrupt=0.7670 loss_corrupt=0.5124 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.6988 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8801 out_g_norm=0.3040 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6686 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=11200 epoch=11200/11500 epoch_step=1/1 micro_steps=11200 elapsed=3.0s lr=2.000000e-03 loss=0.4994 loss_recon=0.4994 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7715 corrupt_frac=1.0000 acc_corrupt=0.7715 loss_corrupt=0.4994 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7020 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8867 out_g_norm=0.2956 acc_corrupt_t_0p6_0p8=0.9606 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5775 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=11250 epoch=11250/11500 epoch_step=1/1 micro_steps=11250 elapsed=3.0s lr=2.000000e-03 loss=0.4923 loss_recon=0.4923 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4923 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7043 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8912 out_g_norm=0.2849 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4828 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=11300 epoch=11300/11500 epoch_step=1/1 micro_steps=11300 elapsed=3.0s lr=2.000000e-03 loss=0.4947 loss_recon=0.4947 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7728 corrupt_frac=1.0000 acc_corrupt=0.7728 loss_corrupt=0.4947 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7021 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9176 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.9035 out_g_norm=0.2545 loss_all=0.5966 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=11350 epoch=11350/11500 epoch_step=1/1 micro_steps=11350 elapsed=3.0s lr=2.000000e-03 loss=0.4847 loss_recon=0.4847 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7743 corrupt_frac=1.0000 acc_corrupt=0.7743 loss_corrupt=0.4847 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7111 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9141 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9596 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.9060 out_g_norm=0.2664 loss_all=0.4626 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=11400 epoch=11400/11500 epoch_step=1/1 micro_steps=11400 elapsed=3.0s lr=2.000000e-03 loss=0.4981 loss_recon=0.4981 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7712 corrupt_frac=1.0000 acc_corrupt=0.7712 loss_corrupt=0.4981 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7029 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9096 out_g_norm=0.2545 loss_all=0.4619 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=11450 epoch=11450/11500 epoch_step=1/1 micro_steps=11450 elapsed=3.0s lr=2.000000e-03 loss=0.5146 loss_recon=0.5146 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7674 corrupt_frac=1.0000 acc_corrupt=0.7674 loss_corrupt=0.5146 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6944 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9164 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.9014 out_g_norm=0.3316 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5327 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=11500 epoch=11500/11500 epoch_step=1/1 micro_steps=11500 elapsed=3.0s lr=2.000000e-03 loss=0.4887 loss_recon=0.4887 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7732 corrupt_frac=1.0000 acc_corrupt=0.7732 loss_corrupt=0.4887 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7067 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8891 out_g_norm=0.2843 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5172 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=11501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=11550 epoch=11550/12000 epoch_step=1/1 micro_steps=11550 elapsed=3.7s lr=2.000000e-03 loss=0.4886 loss_recon=0.4886 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4886 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7111 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8404 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9010 out_g_norm=0.2764 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4931 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=11600 epoch=11600/12000 epoch_step=1/1 micro_steps=11600 elapsed=3.2s lr=2.000000e-03 loss=0.4793 loss_recon=0.4793 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7759 corrupt_frac=1.0000 acc_corrupt=0.7759 loss_corrupt=0.4793 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7092 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.9115 out_g_norm=0.2388 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7138 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=11650 epoch=11650/12000 epoch_step=1/1 micro_steps=11650 elapsed=3.1s lr=2.000000e-03 loss=0.4927 loss_recon=0.4927 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7725 corrupt_frac=1.0000 acc_corrupt=0.7725 loss_corrupt=0.4927 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7081 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.9108 out_g_norm=0.2415 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5463 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=11700 epoch=11700/12000 epoch_step=1/1 micro_steps=11700 elapsed=3.1s lr=2.000000e-03 loss=0.4785 loss_recon=0.4785 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7763 corrupt_frac=1.0000 acc_corrupt=0.7763 loss_corrupt=0.4785 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7103 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.9219 out_g_norm=0.2585 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6188 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=11750 epoch=11750/12000 epoch_step=1/1 micro_steps=11750 elapsed=3.1s lr=2.000000e-03 loss=0.4836 loss_recon=0.4836 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7764 corrupt_frac=1.0000 acc_corrupt=0.7764 loss_corrupt=0.4836 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7094 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9646 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.9186 out_g_norm=0.2831 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5345 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=11800 epoch=11800/12000 epoch_step=1/1 micro_steps=11800 elapsed=3.1s lr=2.000000e-03 loss=0.4794 loss_recon=0.4794 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7770 corrupt_frac=1.0000 acc_corrupt=0.7770 loss_corrupt=0.4794 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7098 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9172 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.9242 out_g_norm=0.2663 loss_all=0.5066 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=11850 epoch=11850/12000 epoch_step=1/1 micro_steps=11850 elapsed=3.1s lr=2.000000e-03 loss=0.4895 loss_recon=0.4895 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4895 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7095 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9147 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9586 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.9293 out_g_norm=0.2773 loss_all=0.4707 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=11900 epoch=11900/12000 epoch_step=1/1 micro_steps=11900 elapsed=3.1s lr=2.000000e-03 loss=0.4823 loss_recon=0.4823 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7750 corrupt_frac=1.0000 acc_corrupt=0.7750 loss_corrupt=0.4823 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7099 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9340 out_g_norm=0.2553 loss_all=0.4649 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=11950 epoch=11950/12000 epoch_step=1/1 micro_steps=11950 elapsed=3.1s lr=2.000000e-03 loss=0.4994 loss_recon=0.4994 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7708 corrupt_frac=1.0000 acc_corrupt=0.7708 loss_corrupt=0.4994 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7003 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.9282 out_g_norm=0.2436 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5076 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=12000 epoch=12000/12000 epoch_step=1/1 micro_steps=12000 elapsed=3.1s lr=2.000000e-03 loss=0.4869 loss_recon=0.4869 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4869 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7076 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.9206 out_g_norm=0.2918 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5477 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=12001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=12050 epoch=12050/12500 epoch_step=1/1 micro_steps=12050 elapsed=3.7s lr=2.000000e-03 loss=0.4945 loss_recon=0.4945 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7718 corrupt_frac=1.0000 acc_corrupt=0.7718 loss_corrupt=0.4945 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7079 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8406 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9082 out_g_norm=0.2682 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5261 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=12100 epoch=12100/12500 epoch_step=1/1 micro_steps=12100 elapsed=3.0s lr=2.000000e-03 loss=0.4789 loss_recon=0.4789 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7772 corrupt_frac=1.0000 acc_corrupt=0.7772 loss_corrupt=0.4789 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7120 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.9004 out_g_norm=0.2577 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5986 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=12150 epoch=12150/12500 epoch_step=1/1 micro_steps=12150 elapsed=3.0s lr=2.000000e-03 loss=0.4975 loss_recon=0.4975 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7705 corrupt_frac=1.0000 acc_corrupt=0.7705 loss_corrupt=0.4975 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7046 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8952 out_g_norm=0.2639 acc_corrupt_t_0p8_1p0=0.9889 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4931 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=12200 epoch=12200/12500 epoch_step=1/1 micro_steps=12200 elapsed=3.0s lr=2.000000e-03 loss=0.4838 loss_recon=0.4838 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7752 corrupt_frac=1.0000 acc_corrupt=0.7752 loss_corrupt=0.4838 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7086 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8975 out_g_norm=0.2709 acc_corrupt_t_0p6_0p8=0.9607 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5803 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=12250 epoch=12250/12500 epoch_step=1/1 micro_steps=12250 elapsed=3.0s lr=2.000000e-03 loss=0.4836 loss_recon=0.4836 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7752 corrupt_frac=1.0000 acc_corrupt=0.7752 loss_corrupt=0.4836 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7076 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.9024 out_g_norm=0.2466 acc_corrupt_t_0p8_1p0=0.9863 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6255 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=12300 epoch=12300/12500 epoch_step=1/1 micro_steps=12300 elapsed=3.0s lr=2.000000e-03 loss=0.4869 loss_recon=0.4869 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7757 corrupt_frac=1.0000 acc_corrupt=0.7757 loss_corrupt=0.4869 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7074 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9609 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.9120 out_g_norm=0.2510 loss_all=0.5664 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=12350 epoch=12350/12500 epoch_step=1/1 micro_steps=12350 elapsed=3.0s lr=2.000000e-03 loss=0.4860 loss_recon=0.4860 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7737 corrupt_frac=1.0000 acc_corrupt=0.7737 loss_corrupt=0.4860 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7102 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9582 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.9068 out_g_norm=0.2436 loss_all=0.5054 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=12400 epoch=12400/12500 epoch_step=1/1 micro_steps=12400 elapsed=3.0s lr=2.000000e-03 loss=0.4944 loss_recon=0.4944 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7720 corrupt_frac=1.0000 acc_corrupt=0.7720 loss_corrupt=0.4944 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7045 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9037 out_g_norm=0.2706 loss_all=0.4927 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=12450 epoch=12450/12500 epoch_step=1/1 micro_steps=12450 elapsed=3.0s lr=2.000000e-03 loss=0.4930 loss_recon=0.4930 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7728 corrupt_frac=1.0000 acc_corrupt=0.7728 loss_corrupt=0.4930 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7041 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.9041 out_g_norm=0.2861 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5210 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=12500 epoch=12500/12500 epoch_step=1/1 micro_steps=12500 elapsed=3.0s lr=2.000000e-03 loss=0.4963 loss_recon=0.4963 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7710 corrupt_frac=1.0000 acc_corrupt=0.7710 loss_corrupt=0.4963 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7029 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8942 out_g_norm=0.3171 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5204 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=12501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=12550 epoch=12550/13000 epoch_step=1/1 micro_steps=12550 elapsed=3.7s lr=2.000000e-03 loss=0.4900 loss_recon=0.4900 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7729 corrupt_frac=1.0000 acc_corrupt=0.7729 loss_corrupt=0.4900 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7095 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8411 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8914 out_g_norm=0.2982 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5186 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=12600 epoch=12600/13000 epoch_step=1/1 micro_steps=12600 elapsed=3.0s lr=2.000000e-03 loss=0.4857 loss_recon=0.4857 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7750 corrupt_frac=1.0000 acc_corrupt=0.7750 loss_corrupt=0.4857 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7080 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9147 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8977 out_g_norm=0.2744 acc_corrupt_t_0p6_0p8=0.9604 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9746 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5972 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=12650 epoch=12650/13000 epoch_step=1/1 micro_steps=12650 elapsed=3.0s lr=2.000000e-03 loss=0.4979 loss_recon=0.4979 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7701 corrupt_frac=1.0000 acc_corrupt=0.7701 loss_corrupt=0.4979 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7038 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8946 out_g_norm=0.2569 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5616 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=12700 epoch=12700/13000 epoch_step=1/1 micro_steps=12700 elapsed=3.0s lr=2.000000e-03 loss=0.4886 loss_recon=0.4886 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7741 corrupt_frac=1.0000 acc_corrupt=0.7741 loss_corrupt=0.4886 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7062 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8965 out_g_norm=0.2542 acc_corrupt_t_0p6_0p8=0.9598 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5989 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=12750 epoch=12750/13000 epoch_step=1/1 micro_steps=12750 elapsed=3.0s lr=2.000000e-03 loss=0.4751 loss_recon=0.4751 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7768 corrupt_frac=1.0000 acc_corrupt=0.7768 loss_corrupt=0.4751 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7104 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9638 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8948 out_g_norm=0.2648 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5617 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=12800 epoch=12800/13000 epoch_step=1/1 micro_steps=12800 elapsed=3.0s lr=2.000000e-03 loss=0.4748 loss_recon=0.4748 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7778 corrupt_frac=1.0000 acc_corrupt=0.7778 loss_corrupt=0.4748 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7113 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.9023 out_g_norm=0.2476 loss_all=0.4633 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=12850 epoch=12850/13000 epoch_step=1/1 micro_steps=12850 elapsed=3.0s lr=2.000000e-03 loss=0.4944 loss_recon=0.4944 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7712 corrupt_frac=1.0000 acc_corrupt=0.7712 loss_corrupt=0.4944 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7062 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9136 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9594 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8976 out_g_norm=0.2723 loss_all=0.4368 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=12900 epoch=12900/13000 epoch_step=1/1 micro_steps=12900 elapsed=3.0s lr=2.000000e-03 loss=0.4939 loss_recon=0.4939 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7735 corrupt_frac=1.0000 acc_corrupt=0.7735 loss_corrupt=0.4939 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7073 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8952 out_g_norm=0.2840 loss_all=0.4543 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=12950 epoch=12950/13000 epoch_step=1/1 micro_steps=12950 elapsed=3.0s lr=2.000000e-03 loss=0.5081 loss_recon=0.5081 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7694 corrupt_frac=1.0000 acc_corrupt=0.7694 loss_corrupt=0.5081 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.6979 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9164 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8888 out_g_norm=0.2926 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5730 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=13000 epoch=13000/13000 epoch_step=1/1 micro_steps=13000 elapsed=3.0s lr=2.000000e-03 loss=0.4698 loss_recon=0.4698 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7781 corrupt_frac=1.0000 acc_corrupt=0.7781 loss_corrupt=0.4698 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7152 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8892 out_g_norm=0.2392 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3898 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=13001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=13050 epoch=13050/13500 epoch_step=1/1 micro_steps=13050 elapsed=3.7s lr=2.000000e-03 loss=0.4795 loss_recon=0.4795 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7754 corrupt_frac=1.0000 acc_corrupt=0.7754 loss_corrupt=0.4795 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7139 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.9026 out_g_norm=0.2583 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5095 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=13100 epoch=13100/13500 epoch_step=1/1 micro_steps=13100 elapsed=3.0s lr=2.000000e-03 loss=0.4786 loss_recon=0.4786 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7771 corrupt_frac=1.0000 acc_corrupt=0.7771 loss_corrupt=0.4786 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7115 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9148 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.9018 out_g_norm=0.2549 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7142 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=13150 epoch=13150/13500 epoch_step=1/1 micro_steps=13150 elapsed=3.0s lr=2.000000e-03 loss=0.4845 loss_recon=0.4845 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7734 corrupt_frac=1.0000 acc_corrupt=0.7734 loss_corrupt=0.4845 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7096 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8970 out_g_norm=0.2477 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5564 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=13200 epoch=13200/13500 epoch_step=1/1 micro_steps=13200 elapsed=3.0s lr=2.000000e-03 loss=0.4758 loss_recon=0.4758 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7763 corrupt_frac=1.0000 acc_corrupt=0.7763 loss_corrupt=0.4758 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7102 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9164 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8968 out_g_norm=0.2372 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4680 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=13250 epoch=13250/13500 epoch_step=1/1 micro_steps=13250 elapsed=3.0s lr=2.000000e-03 loss=0.4768 loss_recon=0.4768 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7778 corrupt_frac=1.0000 acc_corrupt=0.7778 loss_corrupt=0.4768 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7119 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9636 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8907 out_g_norm=0.2637 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5564 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=13300 epoch=13300/13500 epoch_step=1/1 micro_steps=13300 elapsed=3.0s lr=2.000000e-03 loss=0.4703 loss_recon=0.4703 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7792 corrupt_frac=1.0000 acc_corrupt=0.7792 loss_corrupt=0.4703 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7139 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8855 out_g_norm=0.2669 loss_all=0.4967 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=13350 epoch=13350/13500 epoch_step=1/1 micro_steps=13350 elapsed=3.0s lr=2.000000e-03 loss=0.4936 loss_recon=0.4936 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7714 corrupt_frac=1.0000 acc_corrupt=0.7714 loss_corrupt=0.4936 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7061 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9594 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8743 out_g_norm=0.2891 loss_all=0.5164 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=13400 epoch=13400/13500 epoch_step=1/1 micro_steps=13400 elapsed=3.0s lr=2.000000e-03 loss=0.4962 loss_recon=0.4962 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7711 corrupt_frac=1.0000 acc_corrupt=0.7711 loss_corrupt=0.4962 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7029 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8724 out_g_norm=0.2665 loss_all=0.4611 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=13450 epoch=13450/13500 epoch_step=1/1 micro_steps=13450 elapsed=3.0s lr=2.000000e-03 loss=0.4896 loss_recon=0.4896 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7727 corrupt_frac=1.0000 acc_corrupt=0.7727 loss_corrupt=0.4896 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7039 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8413 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8738 out_g_norm=0.2712 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5698 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=13500 epoch=13500/13500 epoch_step=1/1 micro_steps=13500 elapsed=3.0s lr=2.000000e-03 loss=0.4749 loss_recon=0.4749 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7761 corrupt_frac=1.0000 acc_corrupt=0.7761 loss_corrupt=0.4749 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7116 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9632 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8750 out_g_norm=0.2806 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5797 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=13501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=13550 epoch=13550/14000 epoch_step=1/1 micro_steps=13550 elapsed=3.9s lr=2.000000e-03 loss=0.4751 loss_recon=0.4751 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7764 corrupt_frac=1.0000 acc_corrupt=0.7764 loss_corrupt=0.4751 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7156 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9640 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8802 out_g_norm=0.2786 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4158 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=13600 epoch=13600/14000 epoch_step=1/1 micro_steps=13600 elapsed=3.1s lr=2.000000e-03 loss=0.4741 loss_recon=0.4741 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7765 corrupt_frac=1.0000 acc_corrupt=0.7765 loss_corrupt=0.4741 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7104 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8821 out_g_norm=0.2403 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5809 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=13650 epoch=13650/14000 epoch_step=1/1 micro_steps=13650 elapsed=3.1s lr=2.000000e-03 loss=0.4842 loss_recon=0.4842 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7748 corrupt_frac=1.0000 acc_corrupt=0.7748 loss_corrupt=0.4842 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7118 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8844 out_g_norm=0.2434 acc_corrupt_t_0p8_1p0=0.9909 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6087 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=13700 epoch=13700/14000 epoch_step=1/1 micro_steps=13700 elapsed=3.1s lr=2.000000e-03 loss=0.4905 loss_recon=0.4905 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7741 corrupt_frac=1.0000 acc_corrupt=0.7741 loss_corrupt=0.4905 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7065 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8751 out_g_norm=0.2760 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4959 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=13750 epoch=13750/14000 epoch_step=1/1 micro_steps=13750 elapsed=3.1s lr=2.000000e-03 loss=0.4595 loss_recon=0.4595 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7812 corrupt_frac=1.0000 acc_corrupt=0.7812 loss_corrupt=0.4595 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7183 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8799 out_g_norm=0.2376 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5082 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=13800 epoch=13800/14000 epoch_step=1/1 micro_steps=13800 elapsed=3.1s lr=2.000000e-03 loss=0.4824 loss_recon=0.4824 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7755 corrupt_frac=1.0000 acc_corrupt=0.7755 loss_corrupt=0.4824 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7071 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9176 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8901 out_g_norm=0.2721 loss_all=0.4772 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=13850 epoch=13850/14000 epoch_step=1/1 micro_steps=13850 elapsed=3.1s lr=2.000000e-03 loss=0.4904 loss_recon=0.4904 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7728 corrupt_frac=1.0000 acc_corrupt=0.7728 loss_corrupt=0.4904 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7087 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9141 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9598 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8837 out_g_norm=0.2520 loss_all=0.4357 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=13900 epoch=13900/14000 epoch_step=1/1 micro_steps=13900 elapsed=3.1s lr=2.000000e-03 loss=0.4901 loss_recon=0.4901 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7740 corrupt_frac=1.0000 acc_corrupt=0.7740 loss_corrupt=0.4901 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7081 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8846 out_g_norm=0.2474 loss_all=0.4830 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=13950 epoch=13950/14000 epoch_step=1/1 micro_steps=13950 elapsed=3.1s lr=2.000000e-03 loss=0.4841 loss_recon=0.4841 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7751 corrupt_frac=1.0000 acc_corrupt=0.7751 loss_corrupt=0.4841 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7080 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9166 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8825 out_g_norm=0.2843 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5011 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=14000 epoch=14000/14000 epoch_step=1/1 micro_steps=14000 elapsed=3.1s lr=2.000000e-03 loss=0.4776 loss_recon=0.4776 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7759 corrupt_frac=1.0000 acc_corrupt=0.7759 loss_corrupt=0.4776 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7114 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9643 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8886 out_g_norm=0.2692 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4170 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=14001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=14050 epoch=14050/14500 epoch_step=1/1 micro_steps=14050 elapsed=3.8s lr=2.000000e-03 loss=0.4879 loss_recon=0.4879 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7731 corrupt_frac=1.0000 acc_corrupt=0.7731 loss_corrupt=0.4879 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7100 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8409 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8792 out_g_norm=0.2660 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5054 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=14100 epoch=14100/14500 epoch_step=1/1 micro_steps=14100 elapsed=3.1s lr=2.000000e-03 loss=0.4753 loss_recon=0.4753 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7769 corrupt_frac=1.0000 acc_corrupt=0.7769 loss_corrupt=0.4753 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7111 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8791 out_g_norm=0.2601 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6361 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=14150 epoch=14150/14500 epoch_step=1/1 micro_steps=14150 elapsed=3.1s lr=2.000000e-03 loss=0.4770 loss_recon=0.4770 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7758 corrupt_frac=1.0000 acc_corrupt=0.7758 loss_corrupt=0.4770 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7137 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8836 out_g_norm=0.2279 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5934 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=14200 epoch=14200/14500 epoch_step=1/1 micro_steps=14200 elapsed=3.1s lr=2.000000e-03 loss=0.4759 loss_recon=0.4759 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7778 corrupt_frac=1.0000 acc_corrupt=0.7778 loss_corrupt=0.4759 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7131 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8894 out_g_norm=0.2434 acc_corrupt_t_0p6_0p8=0.9605 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4889 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=14250 epoch=14250/14500 epoch_step=1/1 micro_steps=14250 elapsed=3.1s lr=2.000000e-03 loss=0.4618 loss_recon=0.4618 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7809 corrupt_frac=1.0000 acc_corrupt=0.7809 loss_corrupt=0.4618 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7176 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8845 out_g_norm=0.2349 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4753 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=14300 epoch=14300/14500 epoch_step=1/1 micro_steps=14300 elapsed=3.1s lr=2.000000e-03 loss=0.4720 loss_recon=0.4720 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7778 corrupt_frac=1.0000 acc_corrupt=0.7778 loss_corrupt=0.4720 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7109 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9173 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8805 out_g_norm=0.2290 loss_all=0.4368 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=14350 epoch=14350/14500 epoch_step=1/1 micro_steps=14350 elapsed=3.1s lr=2.000000e-03 loss=0.4718 loss_recon=0.4718 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7760 corrupt_frac=1.0000 acc_corrupt=0.7760 loss_corrupt=0.4718 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7140 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9595 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8781 out_g_norm=0.2228 loss_all=0.4346 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=14400 epoch=14400/14500 epoch_step=1/1 micro_steps=14400 elapsed=3.1s lr=2.000000e-03 loss=0.4796 loss_recon=0.4796 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7756 corrupt_frac=1.0000 acc_corrupt=0.7756 loss_corrupt=0.4796 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7104 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8701 out_g_norm=0.2236 loss_all=0.4819 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=14450 epoch=14450/14500 epoch_step=1/1 micro_steps=14450 elapsed=3.1s lr=2.000000e-03 loss=0.4781 loss_recon=0.4781 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7770 corrupt_frac=1.0000 acc_corrupt=0.7770 loss_corrupt=0.4781 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7117 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9164 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8618 out_g_norm=0.2727 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9824 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4476 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=14500 epoch=14500/14500 epoch_step=1/1 micro_steps=14500 elapsed=3.1s lr=2.000000e-03 loss=0.4768 loss_recon=0.4768 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7761 corrupt_frac=1.0000 acc_corrupt=0.7761 loss_corrupt=0.4768 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7117 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9642 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8538 out_g_norm=0.2933 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5174 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=14501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=14550 epoch=14550/15000 epoch_step=1/1 micro_steps=14550 elapsed=3.7s lr=2.000000e-03 loss=0.4842 loss_recon=0.4842 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7733 corrupt_frac=1.0000 acc_corrupt=0.7733 loss_corrupt=0.4842 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7100 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8462 out_g_norm=0.2668 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4302 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=14600 epoch=14600/15000 epoch_step=1/1 micro_steps=14600 elapsed=3.0s lr=2.000000e-03 loss=0.4759 loss_recon=0.4759 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7770 corrupt_frac=1.0000 acc_corrupt=0.7770 loss_corrupt=0.4759 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7113 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8475 out_g_norm=0.2579 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6608 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=14650 epoch=14650/15000 epoch_step=1/1 micro_steps=14650 elapsed=3.1s lr=2.000000e-03 loss=0.4803 loss_recon=0.4803 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7737 corrupt_frac=1.0000 acc_corrupt=0.7737 loss_corrupt=0.4803 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7104 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8429 out_g_norm=0.2524 acc_corrupt_t_0p8_1p0=0.9915 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5620 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=14700 epoch=14700/15000 epoch_step=1/1 micro_steps=14700 elapsed=3.2s lr=2.000000e-03 loss=0.4655 loss_recon=0.4655 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7803 corrupt_frac=1.0000 acc_corrupt=0.7803 loss_corrupt=0.4655 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7176 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9166 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8483 out_g_norm=0.2322 acc_corrupt_t_0p6_0p8=0.9596 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5215 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=14750 epoch=14750/15000 epoch_step=1/1 micro_steps=14750 elapsed=3.1s lr=2.000000e-03 loss=0.4715 loss_recon=0.4715 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.4715 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7122 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9646 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8443 out_g_norm=0.2539 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4841 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=14800 epoch=14800/15000 epoch_step=1/1 micro_steps=14800 elapsed=3.1s lr=2.000000e-03 loss=0.4708 loss_recon=0.4708 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7777 corrupt_frac=1.0000 acc_corrupt=0.7777 loss_corrupt=0.4708 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7111 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9172 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9607 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8373 out_g_norm=0.2486 loss_all=0.4548 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=14850 epoch=14850/15000 epoch_step=1/1 micro_steps=14850 elapsed=3.1s lr=2.000000e-03 loss=0.4896 loss_recon=0.4896 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7731 corrupt_frac=1.0000 acc_corrupt=0.7731 loss_corrupt=0.4896 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7091 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9137 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9588 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8374 out_g_norm=0.2469 loss_all=0.5192 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=14900 epoch=14900/15000 epoch_step=1/1 micro_steps=14900 elapsed=3.1s lr=2.000000e-03 loss=0.4670 loss_recon=0.4670 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7791 corrupt_frac=1.0000 acc_corrupt=0.7791 loss_corrupt=0.4670 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7170 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8392 out_g_norm=0.2502 loss_all=0.4348 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=14950 epoch=14950/15000 epoch_step=1/1 micro_steps=14950 elapsed=3.1s lr=2.000000e-03 loss=0.4713 loss_recon=0.4713 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7772 corrupt_frac=1.0000 acc_corrupt=0.7772 loss_corrupt=0.4713 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7115 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8348 out_g_norm=0.2460 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5176 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=15000 epoch=15000/15000 epoch_step=1/1 micro_steps=15000 elapsed=3.1s lr=2.000000e-03 loss=0.4815 loss_recon=0.4815 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7741 corrupt_frac=1.0000 acc_corrupt=0.7741 loss_corrupt=0.4815 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7081 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9645 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8348 out_g_norm=0.2577 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4646 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=15001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=15050 epoch=15050/15500 epoch_step=1/1 micro_steps=15050 elapsed=3.8s lr=2.000000e-03 loss=0.4768 loss_recon=0.4768 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7760 corrupt_frac=1.0000 acc_corrupt=0.7760 loss_corrupt=0.4768 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7146 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8416 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8345 out_g_norm=0.2634 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4648 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=15100 epoch=15100/15500 epoch_step=1/1 micro_steps=15100 elapsed=3.2s lr=2.000000e-03 loss=0.4683 loss_recon=0.4683 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7785 corrupt_frac=1.0000 acc_corrupt=0.7785 loss_corrupt=0.4683 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7143 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9150 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8359 out_g_norm=0.2965 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5119 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=15150 epoch=15150/15500 epoch_step=1/1 micro_steps=15150 elapsed=3.2s lr=2.000000e-03 loss=0.4848 loss_recon=0.4848 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7736 corrupt_frac=1.0000 acc_corrupt=0.7736 loss_corrupt=0.4848 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7104 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8276 out_g_norm=0.2949 acc_corrupt_t_0p8_1p0=0.9889 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4825 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=15200 epoch=15200/15500 epoch_step=1/1 micro_steps=15200 elapsed=3.2s lr=2.000000e-03 loss=0.4774 loss_recon=0.4774 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7768 corrupt_frac=1.0000 acc_corrupt=0.7768 loss_corrupt=0.4774 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7114 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9166 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8344 out_g_norm=0.2468 acc_corrupt_t_0p6_0p8=0.9609 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4746 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=15250 epoch=15250/15500 epoch_step=1/1 micro_steps=15250 elapsed=3.2s lr=2.000000e-03 loss=0.4618 loss_recon=0.4618 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7808 corrupt_frac=1.0000 acc_corrupt=0.7808 loss_corrupt=0.4618 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7170 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9639 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8423 out_g_norm=0.2464 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4477 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=15300 epoch=15300/15500 epoch_step=1/1 micro_steps=15300 elapsed=3.2s lr=2.000000e-03 loss=0.4613 loss_recon=0.4613 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7804 corrupt_frac=1.0000 acc_corrupt=0.7804 loss_corrupt=0.4613 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7159 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9168 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9614 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8465 out_g_norm=0.2552 loss_all=0.4863 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=15350 epoch=15350/15500 epoch_step=1/1 micro_steps=15350 elapsed=3.2s lr=2.000000e-03 loss=0.4785 loss_recon=0.4785 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7753 corrupt_frac=1.0000 acc_corrupt=0.7753 loss_corrupt=0.4785 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7129 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9140 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9595 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8450 out_g_norm=0.2424 loss_all=0.4761 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=15400 epoch=15400/15500 epoch_step=1/1 micro_steps=15400 elapsed=3.2s lr=2.000000e-03 loss=0.4745 loss_recon=0.4745 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7766 corrupt_frac=1.0000 acc_corrupt=0.7766 loss_corrupt=0.4745 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7123 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8362 out_g_norm=0.2431 loss_all=0.4666 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=15450 epoch=15450/15500 epoch_step=1/1 micro_steps=15450 elapsed=3.2s lr=2.000000e-03 loss=0.4861 loss_recon=0.4861 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7757 corrupt_frac=1.0000 acc_corrupt=0.7757 loss_corrupt=0.4861 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7090 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9167 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8421 out_g_norm=0.2825 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5085 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=15500 epoch=15500/15500 epoch_step=1/1 micro_steps=15500 elapsed=3.2s lr=2.000000e-03 loss=0.4831 loss_recon=0.4831 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7751 corrupt_frac=1.0000 acc_corrupt=0.7751 loss_corrupt=0.4831 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7101 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8414 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9644 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8355 out_g_norm=0.2747 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4677 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=15501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=15550 epoch=15550/16000 epoch_step=1/1 micro_steps=15550 elapsed=3.7s lr=2.000000e-03 loss=0.4815 loss_recon=0.4815 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7746 corrupt_frac=1.0000 acc_corrupt=0.7746 loss_corrupt=0.4815 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7125 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8412 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8398 out_g_norm=0.2444 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4517 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=15600 epoch=15600/16000 epoch_step=1/1 micro_steps=15600 elapsed=3.0s lr=2.000000e-03 loss=0.4636 loss_recon=0.4636 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7795 corrupt_frac=1.0000 acc_corrupt=0.7795 loss_corrupt=0.4636 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7158 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8457 out_g_norm=0.2413 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5890 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=15650 epoch=15650/16000 epoch_step=1/1 micro_steps=15650 elapsed=3.0s lr=2.000000e-03 loss=0.4898 loss_recon=0.4898 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7722 corrupt_frac=1.0000 acc_corrupt=0.7722 loss_corrupt=0.4898 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7076 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8419 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8482 out_g_norm=0.2684 acc_corrupt_t_0p8_1p0=0.9876 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4864 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=15700 epoch=15700/16000 epoch_step=1/1 micro_steps=15700 elapsed=3.0s lr=2.000000e-03 loss=0.4707 loss_recon=0.4707 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7777 corrupt_frac=1.0000 acc_corrupt=0.7777 loss_corrupt=0.4707 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7130 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8509 out_g_norm=0.2600 acc_corrupt_t_0p6_0p8=0.9609 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4884 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=15750 epoch=15750/16000 epoch_step=1/1 micro_steps=15750 elapsed=3.0s lr=2.000000e-03 loss=0.4558 loss_recon=0.4558 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7820 corrupt_frac=1.0000 acc_corrupt=0.7820 loss_corrupt=0.4558 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7194 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9642 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8496 out_g_norm=0.2192 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4790 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=15800 epoch=15800/16000 epoch_step=1/1 micro_steps=15800 elapsed=3.0s lr=2.000000e-03 loss=0.4679 loss_recon=0.4679 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7785 corrupt_frac=1.0000 acc_corrupt=0.7785 loss_corrupt=0.4679 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7120 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9174 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8498 out_g_norm=0.2509 loss_all=0.5006 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=15850 epoch=15850/16000 epoch_step=1/1 micro_steps=15850 elapsed=3.0s lr=2.000000e-03 loss=0.4783 loss_recon=0.4783 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7755 corrupt_frac=1.0000 acc_corrupt=0.7755 loss_corrupt=0.4783 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7132 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8462 out_g_norm=0.2693 loss_all=0.5479 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=15900 epoch=15900/16000 epoch_step=1/1 micro_steps=15900 elapsed=3.0s lr=2.000000e-03 loss=0.4756 loss_recon=0.4756 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7763 corrupt_frac=1.0000 acc_corrupt=0.7763 loss_corrupt=0.4756 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7122 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8358 out_g_norm=0.2561 loss_all=0.4436 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=15950 epoch=15950/16000 epoch_step=1/1 micro_steps=15950 elapsed=3.0s lr=2.000000e-03 loss=0.4797 loss_recon=0.4797 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7765 corrupt_frac=1.0000 acc_corrupt=0.7765 loss_corrupt=0.4797 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7102 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8421 out_g_norm=0.2523 acc_corrupt_t_0p6_0p8=0.9611 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5086 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=16000 epoch=16000/16000 epoch_step=1/1 micro_steps=16000 elapsed=3.0s lr=2.000000e-03 loss=0.4699 loss_recon=0.4699 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7787 corrupt_frac=1.0000 acc_corrupt=0.7787 loss_corrupt=0.4699 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7162 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8474 out_g_norm=0.2543 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5045 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=16001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=16050 epoch=16050/16500 epoch_step=1/1 micro_steps=16050 elapsed=3.8s lr=2.000000e-03 loss=0.4610 loss_recon=0.4610 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7785 corrupt_frac=1.0000 acc_corrupt=0.7785 loss_corrupt=0.4610 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7194 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8410 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8401 out_g_norm=0.2424 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3955 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=16100 epoch=16100/16500 epoch_step=1/1 micro_steps=16100 elapsed=3.1s lr=2.000000e-03 loss=0.4608 loss_recon=0.4608 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7799 corrupt_frac=1.0000 acc_corrupt=0.7799 loss_corrupt=0.4608 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7165 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8475 out_g_norm=0.2253 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4825 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=16150 epoch=16150/16500 epoch_step=1/1 micro_steps=16150 elapsed=3.1s lr=2.000000e-03 loss=0.4796 loss_recon=0.4796 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7737 corrupt_frac=1.0000 acc_corrupt=0.7737 loss_corrupt=0.4796 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7099 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8408 out_g_norm=0.2247 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5488 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=16200 epoch=16200/16500 epoch_step=1/1 micro_steps=16200 elapsed=3.1s lr=2.000000e-03 loss=0.4608 loss_recon=0.4608 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7814 corrupt_frac=1.0000 acc_corrupt=0.7814 loss_corrupt=0.4608 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7194 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9167 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8419 out_g_norm=0.2238 acc_corrupt_t_0p6_0p8=0.9617 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5064 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=16250 epoch=16250/16500 epoch_step=1/1 micro_steps=16250 elapsed=3.1s lr=2.000000e-03 loss=0.4611 loss_recon=0.4611 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7818 corrupt_frac=1.0000 acc_corrupt=0.7818 loss_corrupt=0.4611 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7191 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9153 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9642 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8391 out_g_norm=0.2743 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4529 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=16300 epoch=16300/16500 epoch_step=1/1 micro_steps=16300 elapsed=3.1s lr=2.000000e-03 loss=0.4666 loss_recon=0.4666 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7790 corrupt_frac=1.0000 acc_corrupt=0.7790 loss_corrupt=0.4666 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7134 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9172 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8244 out_g_norm=0.2436 loss_all=0.4257 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=16350 epoch=16350/16500 epoch_step=1/1 micro_steps=16350 elapsed=3.1s lr=2.000000e-03 loss=0.4793 loss_recon=0.4793 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7749 corrupt_frac=1.0000 acc_corrupt=0.7749 loss_corrupt=0.4793 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7123 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9138 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9590 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8213 out_g_norm=0.2403 loss_all=0.5396 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=16400 epoch=16400/16500 epoch_step=1/1 micro_steps=16400 elapsed=3.1s lr=2.000000e-03 loss=0.4583 loss_recon=0.4583 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7821 corrupt_frac=1.0000 acc_corrupt=0.7821 loss_corrupt=0.4583 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7219 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8432 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8202 out_g_norm=0.2400 loss_all=0.4198 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=16450 epoch=16450/16500 epoch_step=1/1 micro_steps=16450 elapsed=3.1s lr=2.000000e-03 loss=0.4877 loss_recon=0.4877 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7744 corrupt_frac=1.0000 acc_corrupt=0.7744 loss_corrupt=0.4877 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7065 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8222 out_g_norm=0.2614 acc_corrupt_t_0p6_0p8=0.9619 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4910 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=16500 epoch=16500/16500 epoch_step=1/1 micro_steps=16500 elapsed=3.1s lr=2.000000e-03 loss=0.4681 loss_recon=0.4681 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7778 corrupt_frac=1.0000 acc_corrupt=0.7778 loss_corrupt=0.4681 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7147 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9637 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.8148 out_g_norm=0.2692 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4287 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=16501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=16550 epoch=16550/17000 epoch_step=1/1 micro_steps=16550 elapsed=3.7s lr=2.000000e-03 loss=0.4725 loss_recon=0.4725 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7770 corrupt_frac=1.0000 acc_corrupt=0.7770 loss_corrupt=0.4725 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7165 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8415 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8165 out_g_norm=0.2347 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4158 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=16600 epoch=16600/17000 epoch_step=1/1 micro_steps=16600 elapsed=3.0s lr=2.000000e-03 loss=0.4668 loss_recon=0.4668 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7796 corrupt_frac=1.0000 acc_corrupt=0.7796 loss_corrupt=0.4668 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7162 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9148 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.8191 out_g_norm=0.2680 acc_corrupt_t_0p6_0p8=0.9605 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5753 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=16650 epoch=16650/17000 epoch_step=1/1 micro_steps=16650 elapsed=3.0s lr=2.000000e-03 loss=0.4724 loss_recon=0.4724 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7764 corrupt_frac=1.0000 acc_corrupt=0.7764 loss_corrupt=0.4724 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7145 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.8091 out_g_norm=0.2626 acc_corrupt_t_0p8_1p0=0.9889 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5301 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=16700 epoch=16700/17000 epoch_step=1/1 micro_steps=16700 elapsed=3.0s lr=2.000000e-03 loss=0.4567 loss_recon=0.4567 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7814 corrupt_frac=1.0000 acc_corrupt=0.7814 loss_corrupt=0.4567 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7195 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.8115 out_g_norm=0.2087 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5326 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=16750 epoch=16750/17000 epoch_step=1/1 micro_steps=16750 elapsed=3.0s lr=2.000000e-03 loss=0.4730 loss_recon=0.4730 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7773 corrupt_frac=1.0000 acc_corrupt=0.7773 loss_corrupt=0.4730 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7110 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.8082 out_g_norm=0.2492 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5143 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=16800 epoch=16800/17000 epoch_step=1/1 micro_steps=16800 elapsed=3.0s lr=2.000000e-03 loss=0.4670 loss_recon=0.4670 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7804 corrupt_frac=1.0000 acc_corrupt=0.7804 loss_corrupt=0.4670 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7161 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9168 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.8028 out_g_norm=0.2589 loss_all=0.5149 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=16850 epoch=16850/17000 epoch_step=1/1 micro_steps=16850 elapsed=3.0s lr=2.000000e-03 loss=0.4788 loss_recon=0.4788 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7758 corrupt_frac=1.0000 acc_corrupt=0.7758 loss_corrupt=0.4788 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7139 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9143 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9591 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.8053 out_g_norm=0.2511 loss_all=0.5734 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=16900 epoch=16900/17000 epoch_step=1/1 micro_steps=16900 elapsed=3.0s lr=2.000000e-03 loss=0.4652 loss_recon=0.4652 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7791 corrupt_frac=1.0000 acc_corrupt=0.7791 loss_corrupt=0.4652 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7171 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.8061 out_g_norm=0.2242 loss_all=0.5282 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=16950 epoch=16950/17000 epoch_step=1/1 micro_steps=16950 elapsed=3.0s lr=2.000000e-03 loss=0.4753 loss_recon=0.4753 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7768 corrupt_frac=1.0000 acc_corrupt=0.7768 loss_corrupt=0.4753 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7105 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.8062 out_g_norm=0.2364 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5683 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=17000 epoch=17000/17000 epoch_step=1/1 micro_steps=17000 elapsed=3.0s lr=2.000000e-03 loss=0.4670 loss_recon=0.4670 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.4670 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7148 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9648 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7932 out_g_norm=0.2265 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4479 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=17001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=17050 epoch=17050/17500 epoch_step=1/1 micro_steps=17050 elapsed=3.7s lr=2.000000e-03 loss=0.4499 loss_recon=0.4499 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7818 corrupt_frac=1.0000 acc_corrupt=0.7818 loss_corrupt=0.4499 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7245 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7988 out_g_norm=0.1918 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4090 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=17100 epoch=17100/17500 epoch_step=1/1 micro_steps=17100 elapsed=3.0s lr=2.000000e-03 loss=0.4836 loss_recon=0.4836 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7749 corrupt_frac=1.0000 acc_corrupt=0.7749 loss_corrupt=0.4836 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7072 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8433 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7931 out_g_norm=0.2539 acc_corrupt_t_0p6_0p8=0.9611 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6027 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=17150 epoch=17150/17500 epoch_step=1/1 micro_steps=17150 elapsed=3.0s lr=2.000000e-03 loss=0.4817 loss_recon=0.4817 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7752 corrupt_frac=1.0000 acc_corrupt=0.7752 loss_corrupt=0.4817 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7124 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9629 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7703 out_g_norm=0.2525 acc_corrupt_t_0p8_1p0=0.9928 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4862 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=17200 epoch=17200/17500 epoch_step=1/1 micro_steps=17200 elapsed=3.0s lr=2.000000e-03 loss=0.4625 loss_recon=0.4625 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7797 corrupt_frac=1.0000 acc_corrupt=0.7797 loss_corrupt=0.4625 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7165 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9167 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7764 out_g_norm=0.2458 acc_corrupt_t_0p6_0p8=0.9605 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5260 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=17250 epoch=17250/17500 epoch_step=1/1 micro_steps=17250 elapsed=3.0s lr=2.000000e-03 loss=0.4562 loss_recon=0.4562 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7819 corrupt_frac=1.0000 acc_corrupt=0.7819 loss_corrupt=0.4562 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7197 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9639 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7793 out_g_norm=0.2645 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4182 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=17300 epoch=17300/17500 epoch_step=1/1 micro_steps=17300 elapsed=3.0s lr=2.000000e-03 loss=0.4564 loss_recon=0.4564 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7823 corrupt_frac=1.0000 acc_corrupt=0.7823 loss_corrupt=0.4564 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7192 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9173 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7767 out_g_norm=0.2310 loss_all=0.5294 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=17350 epoch=17350/17500 epoch_step=1/1 micro_steps=17350 elapsed=3.0s lr=2.000000e-03 loss=0.4684 loss_recon=0.4684 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.4684 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7172 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9144 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9595 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7814 out_g_norm=0.2327 loss_all=0.4754 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=17400 epoch=17400/17500 epoch_step=1/1 micro_steps=17400 elapsed=3.0s lr=2.000000e-03 loss=0.4600 loss_recon=0.4600 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7800 corrupt_frac=1.0000 acc_corrupt=0.7800 loss_corrupt=0.4600 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7184 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7758 out_g_norm=0.2342 loss_all=0.4561 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=17450 epoch=17450/17500 epoch_step=1/1 micro_steps=17450 elapsed=3.0s lr=2.000000e-03 loss=0.4832 loss_recon=0.4832 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7756 corrupt_frac=1.0000 acc_corrupt=0.7756 loss_corrupt=0.4832 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7083 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7803 out_g_norm=0.2232 acc_corrupt_t_0p6_0p8=0.9620 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5898 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=17500 epoch=17500/17500 epoch_step=1/1 micro_steps=17500 elapsed=3.0s lr=2.000000e-03 loss=0.4802 loss_recon=0.4802 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7757 corrupt_frac=1.0000 acc_corrupt=0.7757 loss_corrupt=0.4802 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7109 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7775 out_g_norm=0.2570 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4304 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=17501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=17550 epoch=17550/18000 epoch_step=1/1 micro_steps=17550 elapsed=3.7s lr=2.000000e-03 loss=0.4620 loss_recon=0.4620 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7797 corrupt_frac=1.0000 acc_corrupt=0.7797 loss_corrupt=0.4620 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7211 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8417 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9627 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7801 out_g_norm=0.2403 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4114 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=17600 epoch=17600/18000 epoch_step=1/1 micro_steps=17600 elapsed=3.0s lr=2.000000e-03 loss=0.4667 loss_recon=0.4667 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7791 corrupt_frac=1.0000 acc_corrupt=0.7791 loss_corrupt=0.4667 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7151 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9146 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7844 out_g_norm=0.2448 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5665 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=17650 epoch=17650/18000 epoch_step=1/1 micro_steps=17650 elapsed=3.2s lr=2.000000e-03 loss=0.4761 loss_recon=0.4761 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7757 corrupt_frac=1.0000 acc_corrupt=0.7757 loss_corrupt=0.4761 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7131 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7824 out_g_norm=0.2545 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4852 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=17700 epoch=17700/18000 epoch_step=1/1 micro_steps=17700 elapsed=3.1s lr=2.000000e-03 loss=0.4471 loss_recon=0.4471 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7841 corrupt_frac=1.0000 acc_corrupt=0.7841 loss_corrupt=0.4471 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7241 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9161 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7924 out_g_norm=0.2237 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5571 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=17750 epoch=17750/18000 epoch_step=1/1 micro_steps=17750 elapsed=3.1s lr=2.000000e-03 loss=0.4439 loss_recon=0.4439 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7852 corrupt_frac=1.0000 acc_corrupt=0.7852 loss_corrupt=0.4439 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7251 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9643 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7962 out_g_norm=0.2135 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4649 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=17800 epoch=17800/18000 epoch_step=1/1 micro_steps=17800 elapsed=3.1s lr=2.000000e-03 loss=0.4544 loss_recon=0.4544 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7829 corrupt_frac=1.0000 acc_corrupt=0.7829 loss_corrupt=0.4544 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7202 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7967 out_g_norm=0.2312 loss_all=0.5110 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=17850 epoch=17850/18000 epoch_step=1/1 micro_steps=17850 elapsed=3.1s lr=2.000000e-03 loss=0.4799 loss_recon=0.4799 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7753 corrupt_frac=1.0000 acc_corrupt=0.7753 loss_corrupt=0.4799 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7129 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9144 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9595 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7853 out_g_norm=0.2354 loss_all=0.4390 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=17900 epoch=17900/18000 epoch_step=1/1 micro_steps=17900 elapsed=3.1s lr=2.000000e-03 loss=0.4642 loss_recon=0.4642 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7785 corrupt_frac=1.0000 acc_corrupt=0.7785 loss_corrupt=0.4642 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7161 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9628 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7770 out_g_norm=0.2553 loss_all=0.4667 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=17950 epoch=17950/18000 epoch_step=1/1 micro_steps=17950 elapsed=3.1s lr=2.000000e-03 loss=0.4808 loss_recon=0.4808 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7760 corrupt_frac=1.0000 acc_corrupt=0.7760 loss_corrupt=0.4808 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7096 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9166 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7769 out_g_norm=0.2597 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5609 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=18000 epoch=18000/18000 epoch_step=1/1 micro_steps=18000 elapsed=3.1s lr=2.000000e-03 loss=0.4588 loss_recon=0.4588 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7814 corrupt_frac=1.0000 acc_corrupt=0.7814 loss_corrupt=0.4588 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7211 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7723 out_g_norm=0.2585 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4518 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=18001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=18050 epoch=18050/18500 epoch_step=1/1 micro_steps=18050 elapsed=3.7s lr=2.000000e-03 loss=0.4647 loss_recon=0.4647 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7793 corrupt_frac=1.0000 acc_corrupt=0.7793 loss_corrupt=0.4647 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7203 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7787 out_g_norm=0.2265 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4425 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=18100 epoch=18100/18500 epoch_step=1/1 micro_steps=18100 elapsed=3.0s lr=2.000000e-03 loss=0.4630 loss_recon=0.4630 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7809 corrupt_frac=1.0000 acc_corrupt=0.7809 loss_corrupt=0.4630 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7179 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8435 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7770 out_g_norm=0.2231 acc_corrupt_t_0p6_0p8=0.9602 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6355 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=18150 epoch=18150/18500 epoch_step=1/1 micro_steps=18150 elapsed=3.0s lr=2.000000e-03 loss=0.4655 loss_recon=0.4655 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.4655 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7173 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7697 out_g_norm=0.2177 acc_corrupt_t_0p8_1p0=0.9909 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4659 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=18200 epoch=18200/18500 epoch_step=1/1 micro_steps=18200 elapsed=3.0s lr=2.000000e-03 loss=0.4525 loss_recon=0.4525 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7830 corrupt_frac=1.0000 acc_corrupt=0.7830 loss_corrupt=0.4525 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7225 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7829 out_g_norm=0.2181 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4772 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=18250 epoch=18250/18500 epoch_step=1/1 micro_steps=18250 elapsed=3.0s lr=2.000000e-03 loss=0.4467 loss_recon=0.4467 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7844 corrupt_frac=1.0000 acc_corrupt=0.7844 loss_corrupt=0.4467 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7235 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9643 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7836 out_g_norm=0.2245 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3968 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=18300 epoch=18300/18500 epoch_step=1/1 micro_steps=18300 elapsed=3.0s lr=2.000000e-03 loss=0.4458 loss_recon=0.4458 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7849 corrupt_frac=1.0000 acc_corrupt=0.7849 loss_corrupt=0.4458 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7236 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9172 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9608 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7843 out_g_norm=0.2203 loss_all=0.4749 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=18350 epoch=18350/18500 epoch_step=1/1 micro_steps=18350 elapsed=3.0s lr=2.000000e-03 loss=0.4597 loss_recon=0.4597 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7804 corrupt_frac=1.0000 acc_corrupt=0.7804 loss_corrupt=0.4597 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7219 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9139 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9590 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7822 out_g_norm=0.2313 loss_all=0.4636 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=18400 epoch=18400/18500 epoch_step=1/1 micro_steps=18400 elapsed=3.0s lr=2.000000e-03 loss=0.4587 loss_recon=0.4587 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7810 corrupt_frac=1.0000 acc_corrupt=0.7810 loss_corrupt=0.4587 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7202 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7702 out_g_norm=0.2218 loss_all=0.4467 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=18450 epoch=18450/18500 epoch_step=1/1 micro_steps=18450 elapsed=3.0s lr=2.000000e-03 loss=0.4674 loss_recon=0.4674 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7795 corrupt_frac=1.0000 acc_corrupt=0.7795 loss_corrupt=0.4674 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7155 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9170 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7725 out_g_norm=0.2332 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5601 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=18500 epoch=18500/18500 epoch_step=1/1 micro_steps=18500 elapsed=3.0s lr=2.000000e-03 loss=0.4667 loss_recon=0.4667 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7789 corrupt_frac=1.0000 acc_corrupt=0.7789 loss_corrupt=0.4667 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7167 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9165 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7620 out_g_norm=0.2463 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5629 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=18501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=18550 epoch=18550/19000 epoch_step=1/1 micro_steps=18550 elapsed=3.8s lr=2.000000e-03 loss=0.4563 loss_recon=0.4563 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7803 corrupt_frac=1.0000 acc_corrupt=0.7803 loss_corrupt=0.4563 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7221 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9626 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7588 out_g_norm=0.2265 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4070 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=18600 epoch=18600/19000 epoch_step=1/1 micro_steps=18600 elapsed=3.1s lr=2.000000e-03 loss=0.4568 loss_recon=0.4568 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7825 corrupt_frac=1.0000 acc_corrupt=0.7825 loss_corrupt=0.4568 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7208 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8435 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7646 out_g_norm=0.2204 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7197 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=18650 epoch=18650/19000 epoch_step=1/1 micro_steps=18650 elapsed=3.1s lr=2.000000e-03 loss=0.4693 loss_recon=0.4693 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7774 corrupt_frac=1.0000 acc_corrupt=0.7774 loss_corrupt=0.4693 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7162 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7586 out_g_norm=0.2491 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4727 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=18700 epoch=18700/19000 epoch_step=1/1 micro_steps=18700 elapsed=3.1s lr=2.000000e-03 loss=0.4443 loss_recon=0.4443 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7853 corrupt_frac=1.0000 acc_corrupt=0.7853 loss_corrupt=0.4443 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7265 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9158 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7550 out_g_norm=0.2255 acc_corrupt_t_0p6_0p8=0.9601 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4561 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=18750 epoch=18750/19000 epoch_step=1/1 micro_steps=18750 elapsed=3.1s lr=2.000000e-03 loss=0.4578 loss_recon=0.4578 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7819 corrupt_frac=1.0000 acc_corrupt=0.7819 loss_corrupt=0.4578 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7190 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8432 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9151 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9640 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7600 out_g_norm=0.2399 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5393 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=18800 epoch=18800/19000 epoch_step=1/1 micro_steps=18800 elapsed=3.1s lr=2.000000e-03 loss=0.4441 loss_recon=0.4441 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7862 corrupt_frac=1.0000 acc_corrupt=0.7862 loss_corrupt=0.4441 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7258 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9175 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7570 out_g_norm=0.2276 loss_all=0.4617 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=18850 epoch=18850/19000 epoch_step=1/1 micro_steps=18850 elapsed=3.1s lr=2.000000e-03 loss=0.4692 loss_recon=0.4692 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7769 corrupt_frac=1.0000 acc_corrupt=0.7769 loss_corrupt=0.4692 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7158 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8422 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9141 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9588 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7598 out_g_norm=0.2548 loss_all=0.5040 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=18900 epoch=18900/19000 epoch_step=1/1 micro_steps=18900 elapsed=3.1s lr=2.000000e-03 loss=0.4519 loss_recon=0.4519 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7824 corrupt_frac=1.0000 acc_corrupt=0.7824 loss_corrupt=0.4519 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7227 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9154 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9613 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7569 out_g_norm=0.2379 loss_all=0.4476 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=18950 epoch=18950/19000 epoch_step=1/1 micro_steps=18950 elapsed=3.1s lr=2.000000e-03 loss=0.4764 loss_recon=0.4764 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7760 corrupt_frac=1.0000 acc_corrupt=0.7760 loss_corrupt=0.4764 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7090 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7597 out_g_norm=0.2536 acc_corrupt_t_0p6_0p8=0.9623 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5970 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=19000 epoch=19000/19000 epoch_step=1/1 micro_steps=19000 elapsed=3.1s lr=2.000000e-03 loss=0.4717 loss_recon=0.4717 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7769 corrupt_frac=1.0000 acc_corrupt=0.7769 loss_corrupt=0.4717 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7131 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9635 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7508 out_g_norm=0.2532 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4908 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=19001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=19050 epoch=19050/19500 epoch_step=1/1 micro_steps=19050 elapsed=3.8s lr=2.000000e-03 loss=0.4498 loss_recon=0.4498 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7823 corrupt_frac=1.0000 acc_corrupt=0.7823 loss_corrupt=0.4498 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7256 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8420 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9152 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9622 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7538 out_g_norm=0.2200 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4504 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=19100 epoch=19100/19500 epoch_step=1/1 micro_steps=19100 elapsed=3.1s lr=2.000000e-03 loss=0.4581 loss_recon=0.4581 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7813 corrupt_frac=1.0000 acc_corrupt=0.7813 loss_corrupt=0.4581 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7190 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8431 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7541 out_g_norm=0.2239 acc_corrupt_t_0p6_0p8=0.9625 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5856 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=19150 epoch=19150/19500 epoch_step=1/1 micro_steps=19150 elapsed=3.1s lr=2.000000e-03 loss=0.4845 loss_recon=0.4845 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7748 corrupt_frac=1.0000 acc_corrupt=0.7748 loss_corrupt=0.4845 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7115 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9169 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7409 out_g_norm=0.2479 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4826 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=19200 epoch=19200/19500 epoch_step=1/1 micro_steps=19200 elapsed=3.1s lr=2.000000e-03 loss=0.4457 loss_recon=0.4457 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7837 corrupt_frac=1.0000 acc_corrupt=0.7837 loss_corrupt=0.4457 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7236 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9163 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7419 out_g_norm=0.2143 acc_corrupt_t_0p6_0p8=0.9611 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9902 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4733 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=19250 epoch=19250/19500 epoch_step=1/1 micro_steps=19250 elapsed=3.1s lr=2.000000e-03 loss=0.4585 loss_recon=0.4585 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7815 corrupt_frac=1.0000 acc_corrupt=0.7815 loss_corrupt=0.4585 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7184 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8428 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9641 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7540 out_g_norm=0.2839 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5179 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=19300 epoch=19300/19500 epoch_step=1/1 micro_steps=19300 elapsed=3.1s lr=2.000000e-03 loss=0.4472 loss_recon=0.4472 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7854 corrupt_frac=1.0000 acc_corrupt=0.7854 loss_corrupt=0.4472 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7246 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9176 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7515 out_g_norm=0.2422 loss_all=0.4565 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=19350 epoch=19350/19500 epoch_step=1/1 micro_steps=19350 elapsed=3.1s lr=2.000000e-03 loss=0.4563 loss_recon=0.4563 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7804 corrupt_frac=1.0000 acc_corrupt=0.7804 loss_corrupt=0.4563 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7220 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9135 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9588 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7551 out_g_norm=0.2032 loss_all=0.4438 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=19400 epoch=19400/19500 epoch_step=1/1 micro_steps=19400 elapsed=3.1s lr=2.000000e-03 loss=0.4471 loss_recon=0.4471 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7823 corrupt_frac=1.0000 acc_corrupt=0.7823 loss_corrupt=0.4471 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7223 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8433 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7552 out_g_norm=0.2167 loss_all=0.4171 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=19450 epoch=19450/19500 epoch_step=1/1 micro_steps=19450 elapsed=3.1s lr=2.000000e-03 loss=0.4729 loss_recon=0.4729 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7780 corrupt_frac=1.0000 acc_corrupt=0.7780 loss_corrupt=0.4729 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7124 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8429 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7585 out_g_norm=0.2293 acc_corrupt_t_0p6_0p8=0.9621 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5859 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=19500 epoch=19500/19500 epoch_step=1/1 micro_steps=19500 elapsed=3.1s lr=2.000000e-03 loss=0.4548 loss_recon=0.4548 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7817 corrupt_frac=1.0000 acc_corrupt=0.7817 loss_corrupt=0.4548 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7214 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8425 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9640 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7531 out_g_norm=0.2482 acc_corrupt_t_0p8_1p0=0.9912 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4341 init_gold_top10=0.1982 init_gold_top100=0.3551 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=19501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 1635, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 20000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2221184, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len512_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n512_compactv1635_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=19550 epoch=19550/20000 epoch_step=1/1 micro_steps=19550 elapsed=3.9s lr=2.000000e-03 loss=0.4511 loss_recon=0.4511 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7811 corrupt_frac=1.0000 acc_corrupt=0.7811 loss_corrupt=0.4511 wrong_frac=0.7937 init_acc_corrupt=0.1124 acc_corrupt_t_0p0_0p2=0.7236 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8418 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9157 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9624 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7596 out_g_norm=0.2465 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3745 init_gold_top10=0.2100 init_gold_top100=0.3643 +step=19600 epoch=19600/20000 epoch_step=1/1 micro_steps=19600 elapsed=3.1s lr=2.000000e-03 loss=0.4665 loss_recon=0.4665 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7786 corrupt_frac=1.0000 acc_corrupt=0.7786 loss_corrupt=0.4665 wrong_frac=0.7898 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.7136 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8437 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9155 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=11.7590 out_g_norm=0.2239 acc_corrupt_t_0p6_0p8=0.9618 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5997 init_gold_top10=0.1795 init_gold_top100=0.3530 +step=19650 epoch=19650/20000 epoch_step=1/1 micro_steps=19650 elapsed=3.1s lr=2.000000e-03 loss=0.4841 loss_recon=0.4841 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7733 corrupt_frac=1.0000 acc_corrupt=0.7733 loss_corrupt=0.4841 wrong_frac=0.7940 init_acc_corrupt=0.1131 acc_corrupt_t_0p0_0p2=0.7093 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.8424 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9160 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9610 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=11.7458 out_g_norm=0.2795 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4978 init_gold_top10=0.1746 init_gold_top100=0.3573 +step=19700 epoch=19700/20000 epoch_step=1/1 micro_steps=19700 elapsed=3.1s lr=2.000000e-03 loss=0.4403 loss_recon=0.4403 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7860 corrupt_frac=1.0000 acc_corrupt=0.7860 loss_corrupt=0.4403 wrong_frac=0.7901 init_acc_corrupt=0.1174 acc_corrupt_t_0p0_0p2=0.7274 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.8432 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9162 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=11.7606 out_g_norm=0.2204 acc_corrupt_t_0p6_0p8=0.9605 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4551 init_gold_top10=0.1814 init_gold_top100=0.3535 +step=19750 epoch=19750/20000 epoch_step=1/1 micro_steps=19750 elapsed=3.1s lr=2.000000e-03 loss=0.4530 loss_recon=0.4530 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7829 corrupt_frac=1.0000 acc_corrupt=0.7829 loss_corrupt=0.4530 wrong_frac=0.7886 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.7206 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.8436 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9633 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=11.7696 out_g_norm=0.2402 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4832 init_gold_top10=0.1948 init_gold_top100=0.3601 +step=19800 epoch=19800/20000 epoch_step=1/1 micro_steps=19800 elapsed=3.1s lr=2.000000e-03 loss=0.4499 loss_recon=0.4499 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7839 corrupt_frac=1.0000 acc_corrupt=0.7839 loss_corrupt=0.4499 wrong_frac=0.7877 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.7220 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.8426 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9171 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9615 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.7522 out_g_norm=0.2442 loss_all=0.4519 init_gold_top10=0.2042 init_gold_top100=0.3668 +step=19850 epoch=19850/20000 epoch_step=1/1 micro_steps=19850 elapsed=3.1s lr=2.000000e-03 loss=0.4568 loss_recon=0.4568 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7800 corrupt_frac=1.0000 acc_corrupt=0.7800 loss_corrupt=0.4568 wrong_frac=0.7934 init_acc_corrupt=0.1129 acc_corrupt_t_0p0_0p2=0.7210 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.8427 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9141 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=11.7439 out_g_norm=0.2438 loss_all=0.4615 init_gold_top10=0.2139 init_gold_top100=0.3707 +step=19900 epoch=19900/20000 epoch_step=1/1 micro_steps=19900 elapsed=3.1s lr=2.000000e-03 loss=0.4507 loss_recon=0.4507 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7824 corrupt_frac=1.0000 acc_corrupt=0.7824 loss_corrupt=0.4507 wrong_frac=0.7918 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.7227 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.8430 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9156 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=11.7439 out_g_norm=0.2444 loss_all=0.4229 init_gold_top10=0.1982 init_gold_top100=0.3656 +step=19950 epoch=19950/20000 epoch_step=1/1 micro_steps=19950 elapsed=3.1s lr=2.000000e-03 loss=0.4679 loss_recon=0.4679 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7784 corrupt_frac=1.0000 acc_corrupt=0.7784 loss_corrupt=0.4679 wrong_frac=0.7894 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.7137 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.8421 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9167 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=11.7451 out_g_norm=0.2502 acc_corrupt_t_0p6_0p8=0.9634 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4367 init_gold_top10=0.1817 init_gold_top100=0.3536 +step=20000 epoch=20000/20000 epoch_step=1/1 micro_steps=20000 elapsed=3.1s lr=2.000000e-03 loss=0.4542 loss_recon=0.4542 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7820 corrupt_frac=1.0000 acc_corrupt=0.7820 loss_corrupt=0.4542 wrong_frac=0.7910 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.7221 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.8423 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9159 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9631 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.7453 out_g_norm=0.2320 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4334 init_gold_top10=0.1982 init_gold_top100=0.3551 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt.log new file mode 100644 index 0000000000000000000000000000000000000000..03ff385a4d44133b57fa6842285331e17c56dffc --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt.log @@ -0,0 +1,2375 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=50 epoch=50/500 epoch_step=1/1 micro_steps=50 elapsed=2.4s lr=2.000000e-03 loss=5.7491 loss_recon=5.7491 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1084 corrupt_frac=1.0000 acc_corrupt=0.1084 loss_corrupt=5.7491 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.0355 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.1530 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.4046 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.6250 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=0.5102 out_g_norm=0.6569 acc_corrupt_t_0p8_1p0=0.7969 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.6493 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=1.8s lr=2.000000e-03 loss=5.5162 loss_recon=5.5162 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1303 corrupt_frac=1.0000 acc_corrupt=0.1303 loss_corrupt=5.5162 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.0431 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.1807 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.4545 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=1.8711 out_g_norm=0.9324 acc_corrupt_t_0p6_0p8=0.6610 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.8438 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.3941 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=150 epoch=150/500 epoch_step=1/1 micro_steps=150 elapsed=1.8s lr=2.000000e-03 loss=5.2029 loss_recon=5.2029 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1360 corrupt_frac=1.0000 acc_corrupt=0.1360 loss_corrupt=5.2029 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.0529 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.1896 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.4707 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.6535 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=3.3053 out_g_norm=0.9695 acc_corrupt_t_0p8_1p0=0.8125 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.1273 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=1.8s lr=2.000000e-03 loss=4.8760 loss_recon=4.8760 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1488 corrupt_frac=1.0000 acc_corrupt=0.1488 loss_corrupt=4.8760 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.0601 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.2038 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.4727 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=4.7427 out_g_norm=0.6034 acc_corrupt_t_0p6_0p8=0.6406 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8906 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.8060 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=250 epoch=250/500 epoch_step=1/1 micro_steps=250 elapsed=1.9s lr=2.000000e-03 loss=4.5048 loss_recon=4.5048 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1647 corrupt_frac=1.0000 acc_corrupt=0.1647 loss_corrupt=4.5048 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.0690 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.2261 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.4872 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.6650 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=5.9979 out_g_norm=0.5479 acc_corrupt_t_0p8_1p0=0.8516 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.2454 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=1.8s lr=2.000000e-03 loss=3.9849 loss_recon=3.9849 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1952 corrupt_frac=1.0000 acc_corrupt=0.1952 loss_corrupt=3.9849 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.0832 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.2746 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.5318 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.6891 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=6.9186 out_g_norm=0.5805 loss_all=3.6768 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=350 epoch=350/500 epoch_step=1/1 micro_steps=350 elapsed=1.9s lr=2.000000e-03 loss=3.4521 loss_recon=3.4521 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2265 corrupt_frac=1.0000 acc_corrupt=0.2265 loss_corrupt=3.4521 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.1008 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.3404 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.5950 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.7491 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=7.5483 out_g_norm=0.5430 loss_all=3.0235 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=1.8s lr=2.000000e-03 loss=2.8415 loss_recon=2.8415 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2865 corrupt_frac=1.0000 acc_corrupt=0.2865 loss_corrupt=2.8415 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.1316 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.4325 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.6792 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.7977 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=8.0021 out_g_norm=0.4993 loss_all=2.5778 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=450 epoch=450/500 epoch_step=1/1 micro_steps=450 elapsed=1.8s lr=2.000000e-03 loss=2.3343 loss_recon=2.3343 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3641 corrupt_frac=1.0000 acc_corrupt=0.3641 loss_corrupt=2.3343 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.1757 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.5470 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.7838 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=8.4098 out_g_norm=0.5293 acc_corrupt_t_0p6_0p8=0.8739 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9531 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.2292 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=1.8s lr=2.000000e-03 loss=1.8968 loss_recon=1.8968 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4539 corrupt_frac=1.0000 acc_corrupt=0.4539 loss_corrupt=1.8968 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.2379 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.6876 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.8810 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9323 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=8.7451 out_g_norm=0.6429 acc_corrupt_t_0p8_1p0=0.9453 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.6909 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=550 epoch=550/1000 epoch_step=1/1 micro_steps=550 elapsed=2.3s lr=2.000000e-03 loss=1.5546 loss_recon=1.5546 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5263 corrupt_frac=1.0000 acc_corrupt=0.5263 loss_corrupt=1.5546 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.2957 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.8016 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9403 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9616 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=8.9990 out_g_norm=0.7460 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2913 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=1.7s lr=2.000000e-03 loss=1.2632 loss_recon=1.2632 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5934 corrupt_frac=1.0000 acc_corrupt=0.5934 loss_corrupt=1.2632 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.3372 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.8905 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9812 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.1903 out_g_norm=0.7771 acc_corrupt_t_0p6_0p8=0.9824 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9531 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.1103 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=650 epoch=650/1000 epoch_step=1/1 micro_steps=650 elapsed=1.7s lr=2.000000e-03 loss=1.1275 loss_recon=1.1275 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6117 corrupt_frac=1.0000 acc_corrupt=0.6117 loss_corrupt=1.1275 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.3641 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9204 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9931 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9898 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.3108 out_g_norm=0.8379 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.2394 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=1.7s lr=2.000000e-03 loss=1.0093 loss_recon=1.0093 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6439 corrupt_frac=1.0000 acc_corrupt=0.6439 loss_corrupt=1.0093 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.3935 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9403 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9954 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.3564 out_g_norm=0.8098 acc_corrupt_t_0p6_0p8=0.9927 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0086 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=750 epoch=750/1000 epoch_step=1/1 micro_steps=750 elapsed=1.7s lr=2.000000e-03 loss=0.9847 loss_recon=0.9847 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6494 corrupt_frac=1.0000 acc_corrupt=0.6494 loss_corrupt=0.9847 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.3962 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9475 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=0.9975 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9955 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.3596 out_g_norm=0.8067 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9828 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=1.7s lr=2.000000e-03 loss=0.9173 loss_recon=0.9173 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6707 corrupt_frac=1.0000 acc_corrupt=0.6707 loss_corrupt=0.9173 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.4260 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9595 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9991 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9965 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.3408 out_g_norm=0.7306 loss_all=0.9251 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=850 epoch=850/1000 epoch_step=1/1 micro_steps=850 elapsed=1.7s lr=2.000000e-03 loss=0.9164 loss_recon=0.9164 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6703 corrupt_frac=1.0000 acc_corrupt=0.6703 loss_corrupt=0.9164 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.4383 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9652 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9992 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9980 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.3213 out_g_norm=0.7167 loss_all=0.8963 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=1.7s lr=2.000000e-03 loss=0.8758 loss_recon=0.8758 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6847 corrupt_frac=1.0000 acc_corrupt=0.6847 loss_corrupt=0.8758 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.4534 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9714 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9979 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.3033 out_g_norm=0.6278 loss_all=0.9339 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=950 epoch=950/1000 epoch_step=1/1 micro_steps=950 elapsed=1.7s lr=2.000000e-03 loss=0.8567 loss_recon=0.8567 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6898 corrupt_frac=1.0000 acc_corrupt=0.6898 loss_corrupt=0.8567 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.4546 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9739 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.2996 out_g_norm=0.6084 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8922 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=1.7s lr=2.000000e-03 loss=0.8427 loss_recon=0.8427 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6951 corrupt_frac=1.0000 acc_corrupt=0.6951 loss_corrupt=0.8427 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.4677 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9762 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9988 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.2926 out_g_norm=0.5845 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7955 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1050 epoch=1050/1500 epoch_step=1/1 micro_steps=1050 elapsed=2.3s lr=2.000000e-03 loss=0.8344 loss_recon=0.8344 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6975 corrupt_frac=1.0000 acc_corrupt=0.6975 loss_corrupt=0.8344 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.4788 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9794 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.2886 out_g_norm=0.5167 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7971 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=1100 epoch=1100/1500 epoch_step=1/1 micro_steps=1100 elapsed=1.6s lr=2.000000e-03 loss=0.8035 loss_recon=0.8035 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7125 corrupt_frac=1.0000 acc_corrupt=0.7125 loss_corrupt=0.8035 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.4913 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9803 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.2832 out_g_norm=0.5174 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8368 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=1150 epoch=1150/1500 epoch_step=1/1 micro_steps=1150 elapsed=1.6s lr=2.000000e-03 loss=0.8067 loss_recon=0.8067 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7079 corrupt_frac=1.0000 acc_corrupt=0.7079 loss_corrupt=0.8067 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.4926 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9850 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.2769 out_g_norm=0.5079 acc_corrupt_t_0p8_1p0=0.9896 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9097 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=1200 epoch=1200/1500 epoch_step=1/1 micro_steps=1200 elapsed=1.6s lr=2.000000e-03 loss=0.7651 loss_recon=0.7651 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7246 corrupt_frac=1.0000 acc_corrupt=0.7246 loss_corrupt=0.7651 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.5090 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9860 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.2801 out_g_norm=0.5218 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7780 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=1250 epoch=1250/1500 epoch_step=1/1 micro_steps=1250 elapsed=1.6s lr=2.000000e-03 loss=0.7657 loss_recon=0.7657 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7255 corrupt_frac=1.0000 acc_corrupt=0.7255 loss_corrupt=0.7657 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.5095 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9851 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.2838 out_g_norm=0.4782 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8091 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=1300 epoch=1300/1500 epoch_step=1/1 micro_steps=1300 elapsed=1.6s lr=2.000000e-03 loss=0.7420 loss_recon=0.7420 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7342 corrupt_frac=1.0000 acc_corrupt=0.7342 loss_corrupt=0.7420 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.5239 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9868 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.2855 out_g_norm=0.4786 loss_all=0.7476 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=1350 epoch=1350/1500 epoch_step=1/1 micro_steps=1350 elapsed=1.6s lr=2.000000e-03 loss=0.7467 loss_recon=0.7467 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7323 corrupt_frac=1.0000 acc_corrupt=0.7323 loss_corrupt=0.7467 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.5345 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9863 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.2983 out_g_norm=0.4509 loss_all=0.7189 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=1400 epoch=1400/1500 epoch_step=1/1 micro_steps=1400 elapsed=1.6s lr=2.000000e-03 loss=0.7329 loss_recon=0.7329 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7360 corrupt_frac=1.0000 acc_corrupt=0.7360 loss_corrupt=0.7329 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.5329 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9906 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.3109 out_g_norm=0.4241 loss_all=0.8090 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=1450 epoch=1450/1500 epoch_step=1/1 micro_steps=1450 elapsed=1.6s lr=2.000000e-03 loss=0.7305 loss_recon=0.7305 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7374 corrupt_frac=1.0000 acc_corrupt=0.7374 loss_corrupt=0.7305 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.5290 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9921 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.3218 out_g_norm=0.4085 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8117 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=1500 epoch=1500/1500 epoch_step=1/1 micro_steps=1500 elapsed=1.6s lr=2.000000e-03 loss=0.7098 loss_recon=0.7098 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7422 corrupt_frac=1.0000 acc_corrupt=0.7422 loss_corrupt=0.7098 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.5424 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9914 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.3271 out_g_norm=0.3952 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6973 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=1501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=1550 epoch=1550/2000 epoch_step=1/1 micro_steps=1550 elapsed=2.3s lr=2.000000e-03 loss=0.7188 loss_recon=0.7188 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7420 corrupt_frac=1.0000 acc_corrupt=0.7420 loss_corrupt=0.7188 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.5491 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9928 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.3301 out_g_norm=0.4048 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6760 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=1.7s lr=2.000000e-03 loss=0.7020 loss_recon=0.7020 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7476 corrupt_frac=1.0000 acc_corrupt=0.7476 loss_corrupt=0.7020 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.5477 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9914 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.3407 out_g_norm=0.3934 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7531 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=1650 epoch=1650/2000 epoch_step=1/1 micro_steps=1650 elapsed=1.7s lr=2.000000e-03 loss=0.7077 loss_recon=0.7077 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7441 corrupt_frac=1.0000 acc_corrupt=0.7441 loss_corrupt=0.7077 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.5506 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9945 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.3481 out_g_norm=0.3747 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8136 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=1.7s lr=2.000000e-03 loss=0.6670 loss_recon=0.6670 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7591 corrupt_frac=1.0000 acc_corrupt=0.7591 loss_corrupt=0.6670 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.5664 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9938 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.3589 out_g_norm=0.3663 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6710 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=1750 epoch=1750/2000 epoch_step=1/1 micro_steps=1750 elapsed=1.6s lr=2.000000e-03 loss=0.6802 loss_recon=0.6802 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7554 corrupt_frac=1.0000 acc_corrupt=0.7554 loss_corrupt=0.6802 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.5595 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9919 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.3648 out_g_norm=0.3753 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7146 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=1.6s lr=2.000000e-03 loss=0.6642 loss_recon=0.6642 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7621 corrupt_frac=1.0000 acc_corrupt=0.7621 loss_corrupt=0.6642 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.5704 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9933 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.3725 out_g_norm=0.3766 loss_all=0.6663 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=1850 epoch=1850/2000 epoch_step=1/1 micro_steps=1850 elapsed=1.6s lr=2.000000e-03 loss=0.6739 loss_recon=0.6739 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7574 corrupt_frac=1.0000 acc_corrupt=0.7574 loss_corrupt=0.6739 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.5747 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9930 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.3896 out_g_norm=0.3323 loss_all=0.6682 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=1.6s lr=2.000000e-03 loss=0.6695 loss_recon=0.6695 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7616 corrupt_frac=1.0000 acc_corrupt=0.7616 loss_corrupt=0.6695 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.5769 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9935 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.4034 out_g_norm=0.3641 loss_all=0.7284 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=1950 epoch=1950/2000 epoch_step=1/1 micro_steps=1950 elapsed=1.6s lr=2.000000e-03 loss=0.6655 loss_recon=0.6655 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7604 corrupt_frac=1.0000 acc_corrupt=0.7604 loss_corrupt=0.6655 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.5700 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9931 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.4114 out_g_norm=0.3430 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7118 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=1.6s lr=2.000000e-03 loss=0.6459 loss_recon=0.6459 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7673 corrupt_frac=1.0000 acc_corrupt=0.7673 loss_corrupt=0.6459 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.5853 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9948 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.4215 out_g_norm=0.3322 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6409 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2050 epoch=2050/2500 epoch_step=1/1 micro_steps=2050 elapsed=2.4s lr=2.000000e-03 loss=0.6621 loss_recon=0.6621 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7629 corrupt_frac=1.0000 acc_corrupt=0.7629 loss_corrupt=0.6621 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.5848 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9946 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.4236 out_g_norm=0.3547 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6245 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=2100 epoch=2100/2500 epoch_step=1/1 micro_steps=2100 elapsed=1.6s lr=2.000000e-03 loss=0.6433 loss_recon=0.6433 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7671 corrupt_frac=1.0000 acc_corrupt=0.7671 loss_corrupt=0.6433 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.5810 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9945 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.4306 out_g_norm=0.3333 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6824 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=2150 epoch=2150/2500 epoch_step=1/1 micro_steps=2150 elapsed=1.6s lr=2.000000e-03 loss=0.6562 loss_recon=0.6562 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7630 corrupt_frac=1.0000 acc_corrupt=0.7630 loss_corrupt=0.6562 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.5829 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9963 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.4390 out_g_norm=0.3266 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7837 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=2200 epoch=2200/2500 epoch_step=1/1 micro_steps=2200 elapsed=1.6s lr=2.000000e-03 loss=0.6147 loss_recon=0.6147 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7794 corrupt_frac=1.0000 acc_corrupt=0.7794 loss_corrupt=0.6147 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6013 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9969 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.4530 out_g_norm=0.3411 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6244 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=2250 epoch=2250/2500 epoch_step=1/1 micro_steps=2250 elapsed=1.6s lr=2.000000e-03 loss=0.6305 loss_recon=0.6305 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7727 corrupt_frac=1.0000 acc_corrupt=0.7727 loss_corrupt=0.6305 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.5894 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9943 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.4640 out_g_norm=0.3119 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6792 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=2300 epoch=2300/2500 epoch_step=1/1 micro_steps=2300 elapsed=1.6s lr=2.000000e-03 loss=0.6201 loss_recon=0.6201 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7779 corrupt_frac=1.0000 acc_corrupt=0.7779 loss_corrupt=0.6201 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.5976 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9958 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.4766 out_g_norm=0.3154 loss_all=0.6277 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=2350 epoch=2350/2500 epoch_step=1/1 micro_steps=2350 elapsed=1.6s lr=2.000000e-03 loss=0.6266 loss_recon=0.6266 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7750 corrupt_frac=1.0000 acc_corrupt=0.7750 loss_corrupt=0.6266 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6048 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9949 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.4964 out_g_norm=0.3241 loss_all=0.6229 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=2400 epoch=2400/2500 epoch_step=1/1 micro_steps=2400 elapsed=1.6s lr=2.000000e-03 loss=0.6207 loss_recon=0.6207 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7783 corrupt_frac=1.0000 acc_corrupt=0.7783 loss_corrupt=0.6207 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6060 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9950 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.5105 out_g_norm=0.3013 loss_all=0.6323 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=2450 epoch=2450/2500 epoch_step=1/1 micro_steps=2450 elapsed=1.6s lr=2.000000e-03 loss=0.6284 loss_recon=0.6284 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7746 corrupt_frac=1.0000 acc_corrupt=0.7746 loss_corrupt=0.6284 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.5951 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9941 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.5173 out_g_norm=0.3165 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6489 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=2500 epoch=2500/2500 epoch_step=1/1 micro_steps=2500 elapsed=1.6s lr=2.000000e-03 loss=0.6086 loss_recon=0.6086 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7798 corrupt_frac=1.0000 acc_corrupt=0.7798 loss_corrupt=0.6086 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6068 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9962 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.5302 out_g_norm=0.2976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5811 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=2501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=2550 epoch=2550/3000 epoch_step=1/1 micro_steps=2550 elapsed=2.4s lr=2.000000e-03 loss=0.6225 loss_recon=0.6225 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7783 corrupt_frac=1.0000 acc_corrupt=0.7783 loss_corrupt=0.6225 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6112 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9959 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.5376 out_g_norm=0.2890 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6104 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=1.7s lr=2.000000e-03 loss=0.6058 loss_recon=0.6058 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7817 corrupt_frac=1.0000 acc_corrupt=0.7817 loss_corrupt=0.6058 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6073 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9947 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.5500 out_g_norm=0.3089 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6629 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=2650 epoch=2650/3000 epoch_step=1/1 micro_steps=2650 elapsed=1.7s lr=2.000000e-03 loss=0.6295 loss_recon=0.6295 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7731 corrupt_frac=1.0000 acc_corrupt=0.7731 loss_corrupt=0.6295 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6001 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9973 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.5566 out_g_norm=0.3325 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8041 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=1.7s lr=2.000000e-03 loss=0.5873 loss_recon=0.5873 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7891 corrupt_frac=1.0000 acc_corrupt=0.7891 loss_corrupt=0.5873 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6186 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9974 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.5666 out_g_norm=0.3263 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5847 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=2750 epoch=2750/3000 epoch_step=1/1 micro_steps=2750 elapsed=1.7s lr=2.000000e-03 loss=0.5902 loss_recon=0.5902 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7877 corrupt_frac=1.0000 acc_corrupt=0.7877 loss_corrupt=0.5902 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6154 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9963 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.5744 out_g_norm=0.2843 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6196 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=1.7s lr=2.000000e-03 loss=0.5894 loss_recon=0.5894 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7869 corrupt_frac=1.0000 acc_corrupt=0.7869 loss_corrupt=0.5894 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6140 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9958 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.5858 out_g_norm=0.3065 loss_all=0.5919 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=2850 epoch=2850/3000 epoch_step=1/1 micro_steps=2850 elapsed=1.7s lr=2.000000e-03 loss=0.5860 loss_recon=0.5860 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7883 corrupt_frac=1.0000 acc_corrupt=0.7883 loss_corrupt=0.5860 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6279 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9955 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.5975 out_g_norm=0.2699 loss_all=0.5878 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=1.7s lr=2.000000e-03 loss=0.5844 loss_recon=0.5844 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7912 corrupt_frac=1.0000 acc_corrupt=0.7912 loss_corrupt=0.5844 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6278 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9970 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.6059 out_g_norm=0.2732 loss_all=0.5949 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=2950 epoch=2950/3000 epoch_step=1/1 micro_steps=2950 elapsed=1.6s lr=2.000000e-03 loss=0.5995 loss_recon=0.5995 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7862 corrupt_frac=1.0000 acc_corrupt=0.7862 loss_corrupt=0.5995 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6150 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9958 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.6110 out_g_norm=0.3008 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5847 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=1.6s lr=2.000000e-03 loss=0.5736 loss_recon=0.5736 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7954 corrupt_frac=1.0000 acc_corrupt=0.7954 loss_corrupt=0.5736 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6347 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9965 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.6227 out_g_norm=0.2818 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5873 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3050 epoch=3050/3500 epoch_step=1/1 micro_steps=3050 elapsed=2.3s lr=2.000000e-03 loss=0.5917 loss_recon=0.5917 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7898 corrupt_frac=1.0000 acc_corrupt=0.7898 loss_corrupt=0.5917 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6307 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9971 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.6268 out_g_norm=0.2959 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5892 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=3100 epoch=3100/3500 epoch_step=1/1 micro_steps=3100 elapsed=1.6s lr=2.000000e-03 loss=0.5802 loss_recon=0.5802 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7938 corrupt_frac=1.0000 acc_corrupt=0.7938 loss_corrupt=0.5802 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6294 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9945 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.6407 out_g_norm=0.2891 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5829 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=3150 epoch=3150/3500 epoch_step=1/1 micro_steps=3150 elapsed=1.6s lr=2.000000e-03 loss=0.5990 loss_recon=0.5990 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7834 corrupt_frac=1.0000 acc_corrupt=0.7834 loss_corrupt=0.5990 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6179 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.6472 out_g_norm=0.2858 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7310 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=3200 epoch=3200/3500 epoch_step=1/1 micro_steps=3200 elapsed=1.6s lr=2.000000e-03 loss=0.5567 loss_recon=0.5567 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8014 corrupt_frac=1.0000 acc_corrupt=0.8014 loss_corrupt=0.5567 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6403 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9984 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.6511 out_g_norm=0.2603 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5584 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=3250 epoch=3250/3500 epoch_step=1/1 micro_steps=3250 elapsed=1.6s lr=2.000000e-03 loss=0.5621 loss_recon=0.5621 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7974 corrupt_frac=1.0000 acc_corrupt=0.7974 loss_corrupt=0.5621 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6329 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9965 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.6614 out_g_norm=0.2707 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5801 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=3300 epoch=3300/3500 epoch_step=1/1 micro_steps=3300 elapsed=1.6s lr=2.000000e-03 loss=0.5682 loss_recon=0.5682 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7959 corrupt_frac=1.0000 acc_corrupt=0.7959 loss_corrupt=0.5682 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6311 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9948 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.6662 out_g_norm=0.2901 loss_all=0.6125 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=3350 epoch=3350/3500 epoch_step=1/1 micro_steps=3350 elapsed=1.6s lr=2.000000e-03 loss=0.5618 loss_recon=0.5618 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7998 corrupt_frac=1.0000 acc_corrupt=0.7998 loss_corrupt=0.5618 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6484 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9955 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.6804 out_g_norm=0.2712 loss_all=0.5868 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=3400 epoch=3400/3500 epoch_step=1/1 micro_steps=3400 elapsed=1.6s lr=2.000000e-03 loss=0.5724 loss_recon=0.5724 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7977 corrupt_frac=1.0000 acc_corrupt=0.7977 loss_corrupt=0.5724 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6396 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9966 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.6857 out_g_norm=0.2781 loss_all=0.5641 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=3450 epoch=3450/3500 epoch_step=1/1 micro_steps=3450 elapsed=1.6s lr=2.000000e-03 loss=0.5841 loss_recon=0.5841 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7930 corrupt_frac=1.0000 acc_corrupt=0.7930 loss_corrupt=0.5841 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6271 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9960 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.6984 out_g_norm=0.2879 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6125 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=3500 epoch=3500/3500 epoch_step=1/1 micro_steps=3500 elapsed=1.6s lr=2.000000e-03 loss=0.5602 loss_recon=0.5602 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8053 corrupt_frac=1.0000 acc_corrupt=0.8053 loss_corrupt=0.5602 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6519 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9972 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.7016 out_g_norm=0.2612 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6269 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=3501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=3550 epoch=3550/4000 epoch_step=1/1 micro_steps=3550 elapsed=2.4s lr=2.000000e-03 loss=0.5651 loss_recon=0.5651 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7982 corrupt_frac=1.0000 acc_corrupt=0.7982 loss_corrupt=0.5651 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6464 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9959 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7103 out_g_norm=0.2556 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5314 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=1.7s lr=2.000000e-03 loss=0.5504 loss_recon=0.5504 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8036 corrupt_frac=1.0000 acc_corrupt=0.8036 loss_corrupt=0.5504 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6467 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9954 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.7302 out_g_norm=0.2561 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5603 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=3650 epoch=3650/4000 epoch_step=1/1 micro_steps=3650 elapsed=1.7s lr=2.000000e-03 loss=0.5729 loss_recon=0.5729 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7932 corrupt_frac=1.0000 acc_corrupt=0.7932 loss_corrupt=0.5729 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6359 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9970 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.7329 out_g_norm=0.2716 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7069 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=1.8s lr=2.000000e-03 loss=0.5397 loss_recon=0.5397 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8094 corrupt_frac=1.0000 acc_corrupt=0.8094 loss_corrupt=0.5397 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6554 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9975 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.7485 out_g_norm=0.2361 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5629 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=3750 epoch=3750/4000 epoch_step=1/1 micro_steps=3750 elapsed=1.7s lr=2.000000e-03 loss=0.5467 loss_recon=0.5467 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8047 corrupt_frac=1.0000 acc_corrupt=0.8047 loss_corrupt=0.5467 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6463 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9966 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.7503 out_g_norm=0.2734 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5700 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=1.7s lr=2.000000e-03 loss=0.5374 loss_recon=0.5374 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8071 corrupt_frac=1.0000 acc_corrupt=0.8071 loss_corrupt=0.5374 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6503 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9968 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.7571 out_g_norm=0.2530 loss_all=0.5528 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=3850 epoch=3850/4000 epoch_step=1/1 micro_steps=3850 elapsed=1.7s lr=2.000000e-03 loss=0.5415 loss_recon=0.5415 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8076 corrupt_frac=1.0000 acc_corrupt=0.8076 loss_corrupt=0.5415 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6619 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9957 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.7694 out_g_norm=0.2385 loss_all=0.5405 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=1.7s lr=2.000000e-03 loss=0.5420 loss_recon=0.5420 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8072 corrupt_frac=1.0000 acc_corrupt=0.8072 loss_corrupt=0.5420 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6562 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9974 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7759 out_g_norm=0.2505 loss_all=0.6297 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=3950 epoch=3950/4000 epoch_step=1/1 micro_steps=3950 elapsed=1.7s lr=2.000000e-03 loss=0.5620 loss_recon=0.5620 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8002 corrupt_frac=1.0000 acc_corrupt=0.8002 loss_corrupt=0.5620 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6400 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9965 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.7774 out_g_norm=0.2715 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6011 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=1.7s lr=2.000000e-03 loss=0.5375 loss_recon=0.5375 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8111 corrupt_frac=1.0000 acc_corrupt=0.8111 loss_corrupt=0.5375 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6625 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9971 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.7702 out_g_norm=0.2600 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5416 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4050 epoch=4050/4500 epoch_step=1/1 micro_steps=4050 elapsed=2.4s lr=2.000000e-03 loss=0.5438 loss_recon=0.5438 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8078 corrupt_frac=1.0000 acc_corrupt=0.8078 loss_corrupt=0.5438 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6625 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9974 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7733 out_g_norm=0.2556 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6181 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=4100 epoch=4100/4500 epoch_step=1/1 micro_steps=4100 elapsed=1.6s lr=2.000000e-03 loss=0.5500 loss_recon=0.5500 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8040 corrupt_frac=1.0000 acc_corrupt=0.8040 loss_corrupt=0.5500 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6470 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9959 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.7768 out_g_norm=0.2562 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5289 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=4150 epoch=4150/4500 epoch_step=1/1 micro_steps=4150 elapsed=1.7s lr=2.000000e-03 loss=0.5624 loss_recon=0.5624 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8003 corrupt_frac=1.0000 acc_corrupt=0.8003 loss_corrupt=0.5624 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6478 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9980 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.7802 out_g_norm=0.2791 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6980 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=4200 epoch=4200/4500 epoch_step=1/1 micro_steps=4200 elapsed=1.7s lr=2.000000e-03 loss=0.5018 loss_recon=0.5018 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8196 corrupt_frac=1.0000 acc_corrupt=0.8196 loss_corrupt=0.5018 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6737 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.7899 out_g_norm=0.2394 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5350 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=4250 epoch=4250/4500 epoch_step=1/1 micro_steps=4250 elapsed=1.6s lr=2.000000e-03 loss=0.5188 loss_recon=0.5188 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8133 corrupt_frac=1.0000 acc_corrupt=0.8133 loss_corrupt=0.5188 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6619 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9969 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.7907 out_g_norm=0.2348 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5736 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=4300 epoch=4300/4500 epoch_step=1/1 micro_steps=4300 elapsed=1.6s lr=2.000000e-03 loss=0.5224 loss_recon=0.5224 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8128 corrupt_frac=1.0000 acc_corrupt=0.8128 loss_corrupt=0.5224 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6613 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9959 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.7937 out_g_norm=0.2575 loss_all=0.5467 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=4350 epoch=4350/4500 epoch_step=1/1 micro_steps=4350 elapsed=1.6s lr=2.000000e-03 loss=0.5179 loss_recon=0.5179 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8148 corrupt_frac=1.0000 acc_corrupt=0.8148 loss_corrupt=0.5179 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6743 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9963 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.8055 out_g_norm=0.2534 loss_all=0.5098 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=4400 epoch=4400/4500 epoch_step=1/1 micro_steps=4400 elapsed=1.7s lr=2.000000e-03 loss=0.5207 loss_recon=0.5207 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8182 corrupt_frac=1.0000 acc_corrupt=0.8182 loss_corrupt=0.5207 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6757 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9977 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.8034 out_g_norm=0.2270 loss_all=0.6039 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=4450 epoch=4450/4500 epoch_step=1/1 micro_steps=4450 elapsed=1.6s lr=2.000000e-03 loss=0.5405 loss_recon=0.5405 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8071 corrupt_frac=1.0000 acc_corrupt=0.8071 loss_corrupt=0.5405 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6522 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9968 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.8084 out_g_norm=0.2676 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5870 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=4500 epoch=4500/4500 epoch_step=1/1 micro_steps=4500 elapsed=1.7s lr=2.000000e-03 loss=0.5210 loss_recon=0.5210 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8153 corrupt_frac=1.0000 acc_corrupt=0.8153 loss_corrupt=0.5210 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6713 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9952 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.8124 out_g_norm=0.2484 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5222 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=4501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=4550 epoch=4550/5000 epoch_step=1/1 micro_steps=4550 elapsed=2.4s lr=2.000000e-03 loss=0.5285 loss_recon=0.5285 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8109 corrupt_frac=1.0000 acc_corrupt=0.8109 loss_corrupt=0.5285 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6683 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9968 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.8066 out_g_norm=0.2699 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5267 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=1.7s lr=2.000000e-03 loss=0.5120 loss_recon=0.5120 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8164 corrupt_frac=1.0000 acc_corrupt=0.8164 loss_corrupt=0.5120 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6686 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9973 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.8103 out_g_norm=0.2308 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4449 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=4650 epoch=4650/5000 epoch_step=1/1 micro_steps=4650 elapsed=1.8s lr=2.000000e-03 loss=0.5341 loss_recon=0.5341 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8077 corrupt_frac=1.0000 acc_corrupt=0.8077 loss_corrupt=0.5341 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6612 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9977 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.8186 out_g_norm=0.2387 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6427 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=1.8s lr=2.000000e-03 loss=0.4929 loss_recon=0.4929 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8220 corrupt_frac=1.0000 acc_corrupt=0.8220 loss_corrupt=0.4929 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6783 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9975 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.8322 out_g_norm=0.2470 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5369 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=4750 epoch=4750/5000 epoch_step=1/1 micro_steps=4750 elapsed=1.7s lr=2.000000e-03 loss=0.5146 loss_recon=0.5146 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8145 corrupt_frac=1.0000 acc_corrupt=0.8145 loss_corrupt=0.5146 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6648 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9957 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.8298 out_g_norm=0.2392 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5427 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=1.7s lr=2.000000e-03 loss=0.5034 loss_recon=0.5034 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8206 corrupt_frac=1.0000 acc_corrupt=0.8206 loss_corrupt=0.5034 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6753 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9960 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.8145 out_g_norm=0.2153 loss_all=0.5200 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=4850 epoch=4850/5000 epoch_step=1/1 micro_steps=4850 elapsed=1.7s lr=2.000000e-03 loss=0.4971 loss_recon=0.4971 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8215 corrupt_frac=1.0000 acc_corrupt=0.8215 loss_corrupt=0.4971 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6851 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9980 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.8221 out_g_norm=0.2087 loss_all=0.4896 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=1.7s lr=2.000000e-03 loss=0.5008 loss_recon=0.5008 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8231 corrupt_frac=1.0000 acc_corrupt=0.8231 loss_corrupt=0.5008 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6845 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9975 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.8158 out_g_norm=0.2325 loss_all=0.5707 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=4950 epoch=4950/5000 epoch_step=1/1 micro_steps=4950 elapsed=1.7s lr=2.000000e-03 loss=0.5174 loss_recon=0.5174 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8144 corrupt_frac=1.0000 acc_corrupt=0.8144 loss_corrupt=0.5174 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6659 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9962 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.8150 out_g_norm=0.2241 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5192 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=1.7s lr=2.000000e-03 loss=0.5107 loss_recon=0.5107 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8183 corrupt_frac=1.0000 acc_corrupt=0.8183 loss_corrupt=0.5107 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6756 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9968 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.8033 out_g_norm=0.2318 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4832 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5050 epoch=5050/5500 epoch_step=1/1 micro_steps=5050 elapsed=2.3s lr=2.000000e-03 loss=0.5139 loss_recon=0.5139 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8188 corrupt_frac=1.0000 acc_corrupt=0.8188 loss_corrupt=0.5139 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6827 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9959 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7927 out_g_norm=0.2323 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4899 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=5100 epoch=5100/5500 epoch_step=1/1 micro_steps=5100 elapsed=1.7s lr=2.000000e-03 loss=0.4956 loss_recon=0.4956 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8219 corrupt_frac=1.0000 acc_corrupt=0.8219 loss_corrupt=0.4956 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6794 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9961 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.7873 out_g_norm=0.2252 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5193 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=5150 epoch=5150/5500 epoch_step=1/1 micro_steps=5150 elapsed=1.6s lr=2.000000e-03 loss=0.5258 loss_recon=0.5258 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8116 corrupt_frac=1.0000 acc_corrupt=0.8116 loss_corrupt=0.5258 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6679 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.7790 out_g_norm=0.2515 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6665 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=5200 epoch=5200/5500 epoch_step=1/1 micro_steps=5200 elapsed=1.6s lr=2.000000e-03 loss=0.4860 loss_recon=0.4860 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8264 corrupt_frac=1.0000 acc_corrupt=0.8264 loss_corrupt=0.4860 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6870 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9963 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.7861 out_g_norm=0.2225 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5164 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=5250 epoch=5250/5500 epoch_step=1/1 micro_steps=5250 elapsed=1.6s lr=2.000000e-03 loss=0.5012 loss_recon=0.5012 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8206 corrupt_frac=1.0000 acc_corrupt=0.8206 loss_corrupt=0.5012 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6751 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9969 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.7839 out_g_norm=0.2118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5201 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=5300 epoch=5300/5500 epoch_step=1/1 micro_steps=5300 elapsed=1.6s lr=2.000000e-03 loss=0.4916 loss_recon=0.4916 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8233 corrupt_frac=1.0000 acc_corrupt=0.8233 loss_corrupt=0.4916 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6800 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9967 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.7736 out_g_norm=0.2384 loss_all=0.5166 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=5350 epoch=5350/5500 epoch_step=1/1 micro_steps=5350 elapsed=1.6s lr=2.000000e-03 loss=0.4939 loss_recon=0.4939 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8236 corrupt_frac=1.0000 acc_corrupt=0.8236 loss_corrupt=0.4939 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6901 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9960 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.7803 out_g_norm=0.2263 loss_all=0.4686 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=5400 epoch=5400/5500 epoch_step=1/1 micro_steps=5400 elapsed=1.6s lr=2.000000e-03 loss=0.4924 loss_recon=0.4924 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8239 corrupt_frac=1.0000 acc_corrupt=0.8239 loss_corrupt=0.4924 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6859 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9976 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7713 out_g_norm=0.2316 loss_all=0.5367 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=5450 epoch=5450/5500 epoch_step=1/1 micro_steps=5450 elapsed=1.6s lr=2.000000e-03 loss=0.4934 loss_recon=0.4934 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8246 corrupt_frac=1.0000 acc_corrupt=0.8246 loss_corrupt=0.4934 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6841 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9967 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.7692 out_g_norm=0.2147 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5195 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=5500 epoch=5500/5500 epoch_step=1/1 micro_steps=5500 elapsed=1.8s lr=2.000000e-03 loss=0.4787 loss_recon=0.4787 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8299 corrupt_frac=1.0000 acc_corrupt=0.8299 loss_corrupt=0.4787 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.6957 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9981 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.7664 out_g_norm=0.2153 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4122 init_gold_top10=0.2166 init_gold_top100=0.5586 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt start_step=5501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 335, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1721984, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len64_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n64_compactv335_3l_bs512_hard_ce_allcorrupt/latest.pt" +} +step=5550 epoch=5550/6000 epoch_step=1/1 micro_steps=5550 elapsed=2.5s lr=2.000000e-03 loss=0.4878 loss_recon=0.4878 loss_meanflow=0.0000 mean_model_t=0.2057 mean_corrupt_t=0.2057 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8247 corrupt_frac=1.0000 acc_corrupt=0.8247 loss_corrupt=0.4878 wrong_frac=0.7942 init_acc_corrupt=0.1140 acc_corrupt_t_0p0_0p2=0.6912 corrupt_frac_t_0p0_0p2=0.5666 acc_corrupt_t_0p2_0p4=0.9989 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0731 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7575 out_g_norm=0.2214 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4739 init_gold_top10=0.2269 init_gold_top100=0.5674 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=1.8s lr=2.000000e-03 loss=0.4822 loss_recon=0.4822 loss_meanflow=0.0000 mean_model_t=0.2106 mean_corrupt_t=0.2106 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8282 corrupt_frac=1.0000 acc_corrupt=0.8282 loss_corrupt=0.4822 wrong_frac=0.7895 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.6912 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9957 corrupt_frac_t_0p2_0p4=0.3628 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0772 out_w_norm=9.7544 out_g_norm=0.2400 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6087 init_gold_top10=0.2058 init_gold_top100=0.5554 +step=5650 epoch=5650/6000 epoch_step=1/1 micro_steps=5650 elapsed=1.8s lr=2.000000e-03 loss=0.5256 loss_recon=0.5256 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8138 corrupt_frac=1.0000 acc_corrupt=0.8138 loss_corrupt=0.5256 wrong_frac=0.7943 init_acc_corrupt=0.1145 acc_corrupt_t_0p0_0p2=0.6717 corrupt_frac_t_0p0_0p2=0.5650 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0695 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=9.7439 out_g_norm=0.2479 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5948 init_gold_top10=0.1887 init_gold_top100=0.5629 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=1.7s lr=2.000000e-03 loss=0.4719 loss_recon=0.4719 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8331 corrupt_frac=1.0000 acc_corrupt=0.8331 loss_corrupt=0.4719 wrong_frac=0.7894 init_acc_corrupt=0.1194 acc_corrupt_t_0p0_0p2=0.6984 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9975 corrupt_frac_t_0p2_0p4=0.3647 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 out_w_norm=9.7476 out_g_norm=0.2214 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5138 init_gold_top10=0.1948 init_gold_top100=0.5554 +step=5750 epoch=5750/6000 epoch_step=1/1 micro_steps=5750 elapsed=1.7s lr=2.000000e-03 loss=0.4864 loss_recon=0.4864 loss_meanflow=0.0000 mean_model_t=0.2109 mean_corrupt_t=0.2109 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8268 corrupt_frac=1.0000 acc_corrupt=0.8268 loss_corrupt=0.4864 wrong_frac=0.7892 init_acc_corrupt=0.1201 acc_corrupt_t_0p0_0p2=0.6871 corrupt_frac_t_0p0_0p2=0.5486 acc_corrupt_t_0p2_0p4=0.9957 corrupt_frac_t_0p2_0p4=0.3642 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0148 out_w_norm=9.7375 out_g_norm=0.2191 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5019 init_gold_top10=0.2173 init_gold_top100=0.5640 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=1.7s lr=2.000000e-03 loss=0.4798 loss_recon=0.4798 loss_meanflow=0.0000 mean_model_t=0.2124 mean_corrupt_t=0.2124 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8294 corrupt_frac=1.0000 acc_corrupt=0.8294 loss_corrupt=0.4798 wrong_frac=0.7877 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.6908 corrupt_frac_t_0p0_0p2=0.5483 acc_corrupt_t_0p2_0p4=0.9970 corrupt_frac_t_0p2_0p4=0.3580 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0847 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.7269 out_g_norm=0.2504 loss_all=0.5485 init_gold_top10=0.2249 init_gold_top100=0.5697 +step=5850 epoch=5850/6000 epoch_step=1/1 micro_steps=5850 elapsed=1.7s lr=2.000000e-03 loss=0.4802 loss_recon=0.4802 loss_meanflow=0.0000 mean_model_t=0.2066 mean_corrupt_t=0.2066 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8292 corrupt_frac=1.0000 acc_corrupt=0.8292 loss_corrupt=0.4802 wrong_frac=0.7940 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.6993 corrupt_frac_t_0p0_0p2=0.5647 acc_corrupt_t_0p2_0p4=0.9973 corrupt_frac_t_0p2_0p4=0.3564 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0705 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=9.7335 out_g_norm=0.2183 loss_all=0.4455 init_gold_top10=0.2347 init_gold_top100=0.5719 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=1.7s lr=2.000000e-03 loss=0.4851 loss_recon=0.4851 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8296 corrupt_frac=1.0000 acc_corrupt=0.8296 loss_corrupt=0.4851 wrong_frac=0.7915 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.6972 corrupt_frac_t_0p0_0p2=0.5580 acc_corrupt_t_0p2_0p4=0.9960 corrupt_frac_t_0p2_0p4=0.3578 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0128 out_w_norm=9.7345 out_g_norm=0.2267 loss_all=0.5431 init_gold_top10=0.2195 init_gold_top100=0.5651 +step=5950 epoch=5950/6000 epoch_step=1/1 micro_steps=5950 elapsed=1.7s lr=2.000000e-03 loss=0.4737 loss_recon=0.4737 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8316 corrupt_frac=1.0000 acc_corrupt=0.8316 loss_corrupt=0.4737 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.6957 corrupt_frac_t_0p0_0p2=0.5514 acc_corrupt_t_0p2_0p4=0.9984 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0767 out_w_norm=9.7350 out_g_norm=0.2004 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0146 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4678 init_gold_top10=0.1981 init_gold_top100=0.5691 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=1.7s lr=2.000000e-03 loss=0.4488 loss_recon=0.4488 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8395 corrupt_frac=1.0000 acc_corrupt=0.8395 loss_corrupt=0.4488 wrong_frac=0.7912 init_acc_corrupt=0.1184 acc_corrupt_t_0p0_0p2=0.7147 corrupt_frac_t_0p0_0p2=0.5566 acc_corrupt_t_0p2_0p4=0.9953 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0723 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=9.7340 out_g_norm=0.2080 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3650 init_gold_top10=0.2166 init_gold_top100=0.5586 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_compactv47_3l_hard_ce_onehot.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_compactv47_3l_hard_ce_onehot.log new file mode 100644 index 0000000000000000000000000000000000000000..f939a3dd9a8849ec3cc385e98a4adb10bff3af1a --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_compactv47_3l_hard_ce_onehot.log @@ -0,0 +1,3939 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=50 epoch=25/250 epoch_step=2/2 micro_steps=50 elapsed=5.5s lr=2.000000e-03 loss=3.7839 loss_recon=3.7839 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1725 corrupt_frac=0.7025 acc_corrupt=0.1210 loss_corrupt=3.7839 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.0956 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.1817 out_g_norm=3.1393 acc_corrupt_t_0p2_0p4=0.1429 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.2000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=0.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=3.7441 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=100 epoch=50/250 epoch_step=2/2 micro_steps=100 elapsed=5.4s lr=2.000000e-03 loss=3.5971 loss_recon=3.5971 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1825 corrupt_frac=0.7075 acc_corrupt=0.1237 loss_corrupt=3.5971 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.1378 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.5807 out_g_norm=3.3745 acc_corrupt_t_0p2_0p4=0.0800 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.1667 corrupt_frac_t_0p4_0p6=1.0000 loss_all=3.3760 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=150 epoch=75/250 epoch_step=2/2 micro_steps=150 elapsed=5.4s lr=2.000000e-03 loss=3.2643 loss_recon=3.2643 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3250 corrupt_frac=0.7225 acc_corrupt=0.1834 loss_corrupt=3.2643 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.1560 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=0.9148 out_g_norm=3.5397 acc_corrupt_t_0p0_0p2=0.1871 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.2800 corrupt_frac_t_0p4_0p6=1.0000 loss_all=3.2773 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=200 epoch=100/250 epoch_step=2/2 micro_steps=200 elapsed=4.4s lr=2.000000e-03 loss=2.8517 loss_recon=2.8517 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3350 corrupt_frac=0.6950 acc_corrupt=0.2194 loss_corrupt=2.8517 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.1781 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.2465 out_g_norm=3.6196 acc_corrupt_t_0p2_0p4=0.2661 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.3333 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=0.2353 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.8374 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=250 epoch=125/250 epoch_step=2/2 micro_steps=250 elapsed=1.5s lr=2.000000e-03 loss=2.5747 loss_recon=2.5747 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4000 corrupt_frac=0.7425 acc_corrupt=0.2458 loss_corrupt=2.5747 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.1698 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.5733 out_g_norm=3.5787 acc_corrupt_t_0p2_0p4=0.2869 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.6875 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.9946 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=300 epoch=150/250 epoch_step=2/2 micro_steps=300 elapsed=2.0s lr=2.000000e-03 loss=2.1247 loss_recon=2.1247 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5200 corrupt_frac=0.6800 acc_corrupt=0.3382 loss_corrupt=2.1247 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.2837 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.8719 out_g_norm=3.6981 acc_corrupt_t_0p2_0p4=0.3670 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=0.7500 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.7454 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=350 epoch=175/250 epoch_step=2/2 micro_steps=350 elapsed=1.5s lr=2.000000e-03 loss=2.1176 loss_recon=2.1176 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5225 corrupt_frac=0.7250 acc_corrupt=0.3897 loss_corrupt=2.1176 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.3247 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.1237 out_g_norm=3.8870 acc_corrupt_t_0p4_0p6=0.6429 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.4706 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.7792 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=400 epoch=200/250 epoch_step=2/2 micro_steps=400 elapsed=1.5s lr=2.000000e-03 loss=1.2785 loss_recon=1.2785 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6525 corrupt_frac=0.7075 acc_corrupt=0.5654 loss_corrupt=1.2785 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.6154 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.3247 out_g_norm=3.8331 acc_corrupt_t_0p0_0p2=0.4812 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.8000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.5786 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=450 epoch=225/250 epoch_step=2/2 micro_steps=450 elapsed=1.5s lr=2.000000e-03 loss=1.3615 loss_recon=1.3615 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7125 corrupt_frac=0.6800 acc_corrupt=0.5993 loss_corrupt=1.3615 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.5098 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.5009 out_g_norm=4.0468 acc_corrupt_t_0p2_0p4=0.6962 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.7222 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=2.8559 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=500 epoch=250/250 epoch_step=2/2 micro_steps=500 elapsed=2.3s lr=2.000000e-03 loss=1.0315 loss_recon=1.0315 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7700 corrupt_frac=0.7000 acc_corrupt=0.7107 loss_corrupt=1.0315 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.7135 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.6590 out_g_norm=3.9241 acc_corrupt_t_0p2_0p4=0.6627 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.8462 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.3338 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=550 epoch=275/500 epoch_step=2/2 micro_steps=550 elapsed=4.3s lr=2.000000e-03 loss=0.9164 loss_recon=0.9164 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7775 corrupt_frac=0.7025 acc_corrupt=0.7260 loss_corrupt=0.9164 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.6544 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.7860 out_g_norm=3.9023 acc_corrupt_t_0p2_0p4=0.7647 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.8392 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=600 epoch=300/500 epoch_step=2/2 micro_steps=600 elapsed=1.4s lr=2.000000e-03 loss=0.9140 loss_recon=0.9140 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7900 corrupt_frac=0.7075 acc_corrupt=0.7385 loss_corrupt=0.9140 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.6735 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.8869 out_g_norm=3.6210 acc_corrupt_t_0p2_0p4=0.8667 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0681 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=650 epoch=325/500 epoch_step=2/2 micro_steps=650 elapsed=2.1s lr=2.000000e-03 loss=0.7399 loss_recon=0.7399 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8200 corrupt_frac=0.7225 acc_corrupt=0.7855 loss_corrupt=0.7399 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.7798 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=3.0036 out_g_norm=3.4564 acc_corrupt_t_0p0_0p2=0.7613 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.9600 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.4001 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=700 epoch=350/500 epoch_step=2/2 micro_steps=700 elapsed=1.5s lr=2.000000e-03 loss=0.4556 loss_recon=0.4556 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8900 corrupt_frac=0.6950 acc_corrupt=0.8705 loss_corrupt=0.4556 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.8014 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.1047 out_g_norm=3.1296 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1462 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=750 epoch=375/500 epoch_step=2/2 micro_steps=750 elapsed=1.6s lr=2.000000e-03 loss=0.5879 loss_recon=0.5879 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8425 corrupt_frac=0.7425 acc_corrupt=0.8047 loss_corrupt=0.5879 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.7296 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.2051 out_g_norm=2.9415 acc_corrupt_t_0p2_0p4=0.8852 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9375 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.8400 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=800 epoch=400/500 epoch_step=2/2 micro_steps=800 elapsed=1.4s lr=2.000000e-03 loss=0.5973 loss_recon=0.5973 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8775 corrupt_frac=0.6800 acc_corrupt=0.8419 loss_corrupt=0.5973 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.8298 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.2818 out_g_norm=3.2021 acc_corrupt_t_0p2_0p4=0.8349 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9444 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.7383 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=850 epoch=425/500 epoch_step=2/2 micro_steps=850 elapsed=1.7s lr=2.000000e-03 loss=0.8866 loss_recon=0.8866 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8100 corrupt_frac=0.7250 acc_corrupt=0.7517 loss_corrupt=0.8866 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.6907 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.3566 out_g_norm=3.2124 acc_corrupt_t_0p4_0p6=0.9643 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.8382 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0619 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=900 epoch=450/500 epoch_step=2/2 micro_steps=900 elapsed=4.9s lr=2.000000e-03 loss=0.1975 loss_recon=0.1975 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9300 corrupt_frac=0.7075 acc_corrupt=0.9223 loss_corrupt=0.1975 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9385 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=3.4150 out_g_norm=2.4535 acc_corrupt_t_0p0_0p2=0.8947 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.5414 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=950 epoch=475/500 epoch_step=2/2 micro_steps=950 elapsed=5.1s lr=2.000000e-03 loss=0.7460 loss_recon=0.7460 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8500 corrupt_frac=0.6800 acc_corrupt=0.8051 loss_corrupt=0.7460 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.7124 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.4890 out_g_norm=2.8672 acc_corrupt_t_0p2_0p4=0.8861 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=2.0532 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=1000 epoch=500/500 epoch_step=2/2 micro_steps=1000 elapsed=5.2s lr=2.000000e-03 loss=0.4710 loss_recon=0.4710 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8925 corrupt_frac=0.7000 acc_corrupt=0.8750 loss_corrupt=0.4710 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.8480 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.5646 out_g_norm=2.6766 acc_corrupt_t_0p2_0p4=0.9157 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9231 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0397 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 1500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=1050 epoch=525/750 epoch_step=2/2 micro_steps=1050 elapsed=3.3s lr=2.000000e-03 loss=0.4981 loss_recon=0.4981 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8900 corrupt_frac=0.7025 acc_corrupt=0.8648 loss_corrupt=0.4981 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.7574 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.6252 out_g_norm=2.6030 acc_corrupt_t_0p2_0p4=0.9580 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.2693 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=1100 epoch=550/750 epoch_step=2/2 micro_steps=1100 elapsed=1.3s lr=2.000000e-03 loss=0.4824 loss_recon=0.4824 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8875 corrupt_frac=0.7075 acc_corrupt=0.8693 loss_corrupt=0.4824 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.8520 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.6974 out_g_norm=2.5079 acc_corrupt_t_0p2_0p4=0.8933 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0119 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=1150 epoch=575/750 epoch_step=2/2 micro_steps=1150 elapsed=1.9s lr=2.000000e-03 loss=0.4359 loss_recon=0.4359 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8975 corrupt_frac=0.7225 acc_corrupt=0.8927 loss_corrupt=0.4359 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.8716 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=3.7687 out_g_norm=2.4862 acc_corrupt_t_0p0_0p2=0.8903 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.3966 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=1200 epoch=600/750 epoch_step=2/2 micro_steps=1200 elapsed=1.3s lr=2.000000e-03 loss=0.1668 loss_recon=0.1668 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9525 corrupt_frac=0.6950 acc_corrupt=0.9496 loss_corrupt=0.1668 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9178 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.8311 out_g_norm=2.1724 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=0.9412 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0282 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=1250 epoch=625/750 epoch_step=2/2 micro_steps=1250 elapsed=1.3s lr=2.000000e-03 loss=0.2352 loss_recon=0.2352 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7425 acc_corrupt=0.9327 loss_corrupt=0.2352 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9057 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.8889 out_g_norm=1.8756 acc_corrupt_t_0p2_0p4=0.9590 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.2473 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=1300 epoch=650/750 epoch_step=2/2 micro_steps=1300 elapsed=1.4s lr=2.000000e-03 loss=0.3319 loss_recon=0.3319 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9225 corrupt_frac=0.6800 acc_corrupt=0.8971 loss_corrupt=0.3319 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.8936 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.9531 out_g_norm=1.9334 acc_corrupt_t_0p2_0p4=0.8807 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0836 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=1350 epoch=675/750 epoch_step=2/2 micro_steps=1350 elapsed=1.4s lr=2.000000e-03 loss=0.5472 loss_recon=0.5472 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8625 corrupt_frac=0.7250 acc_corrupt=0.8345 loss_corrupt=0.5472 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.7938 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=3.9984 out_g_norm=2.2414 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.8824 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.3670 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=1400 epoch=700/750 epoch_step=2/2 micro_steps=1400 elapsed=1.4s lr=2.000000e-03 loss=0.0636 loss_recon=0.0636 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7075 acc_corrupt=0.9823 loss_corrupt=0.0636 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9846 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.0354 out_g_norm=1.5721 acc_corrupt_t_0p0_0p2=0.9774 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1129 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=1450 epoch=725/750 epoch_step=2/2 micro_steps=1450 elapsed=1.4s lr=2.000000e-03 loss=0.5187 loss_recon=0.5187 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8950 corrupt_frac=0.6800 acc_corrupt=0.8676 loss_corrupt=0.5187 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.8235 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.0796 out_g_norm=2.0568 acc_corrupt_t_0p2_0p4=0.8861 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=2.3870 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=1500 epoch=750/750 epoch_step=2/2 micro_steps=1500 elapsed=1.4s lr=2.000000e-03 loss=0.3497 loss_recon=0.3497 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9300 corrupt_frac=0.7000 acc_corrupt=0.9179 loss_corrupt=0.3497 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.8889 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.1314 out_g_norm=1.8251 acc_corrupt_t_0p2_0p4=0.9518 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0030 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=1501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=1550 epoch=775/1000 epoch_step=2/2 micro_steps=1550 elapsed=6.0s lr=2.000000e-03 loss=0.3227 loss_recon=0.3227 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9325 corrupt_frac=0.7025 acc_corrupt=0.9181 loss_corrupt=0.3227 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8456 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.1784 out_g_norm=1.7173 acc_corrupt_t_0p2_0p4=0.9832 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.6561 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=1600 epoch=800/1000 epoch_step=2/2 micro_steps=1600 elapsed=1.4s lr=2.000000e-03 loss=0.3053 loss_recon=0.3053 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9300 corrupt_frac=0.7075 acc_corrupt=0.9152 loss_corrupt=0.3053 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9133 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.2248 out_g_norm=1.5357 acc_corrupt_t_0p2_0p4=0.9067 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0161 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=1650 epoch=825/1000 epoch_step=2/2 micro_steps=1650 elapsed=1.5s lr=2.000000e-03 loss=0.3526 loss_recon=0.3526 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9100 corrupt_frac=0.7225 acc_corrupt=0.8893 loss_corrupt=0.3526 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.8716 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.2756 out_g_norm=1.8832 acc_corrupt_t_0p0_0p2=0.8839 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.3357 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=1700 epoch=850/1000 epoch_step=2/2 micro_steps=1700 elapsed=2.0s lr=2.000000e-03 loss=0.1210 loss_recon=0.1210 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6950 acc_corrupt=0.9604 loss_corrupt=0.1210 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9247 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.3085 out_g_norm=1.3790 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0457 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=1750 epoch=875/1000 epoch_step=2/2 micro_steps=1750 elapsed=1.5s lr=2.000000e-03 loss=0.1511 loss_recon=0.1511 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.7425 acc_corrupt=0.9596 loss_corrupt=0.1511 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9245 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.3735 out_g_norm=1.4019 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.2508 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=1800 epoch=900/1000 epoch_step=2/2 micro_steps=1800 elapsed=1.5s lr=2.000000e-03 loss=0.2153 loss_recon=0.2153 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9425 corrupt_frac=0.6800 acc_corrupt=0.9265 loss_corrupt=0.2153 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9433 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.4117 out_g_norm=1.5855 acc_corrupt_t_0p2_0p4=0.8899 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0545 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=1850 epoch=925/1000 epoch_step=2/2 micro_steps=1850 elapsed=1.5s lr=2.000000e-03 loss=0.5287 loss_recon=0.5287 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8900 corrupt_frac=0.7250 acc_corrupt=0.8621 loss_corrupt=0.5287 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.8144 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.4362 out_g_norm=1.5957 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9412 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0036 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=1900 epoch=950/1000 epoch_step=2/2 micro_steps=1900 elapsed=3.0s lr=2.000000e-03 loss=0.0566 loss_recon=0.0566 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9859 loss_corrupt=0.0566 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.4718 out_g_norm=1.1200 acc_corrupt_t_0p0_0p2=0.9699 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0743 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=1950 epoch=975/1000 epoch_step=2/2 micro_steps=1950 elapsed=5.3s lr=2.000000e-03 loss=0.3502 loss_recon=0.3502 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9225 corrupt_frac=0.6800 acc_corrupt=0.8971 loss_corrupt=0.3502 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.8562 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.5097 out_g_norm=1.5773 acc_corrupt_t_0p2_0p4=0.9241 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.2241 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=2000 epoch=1000/1000 epoch_step=2/2 micro_steps=2000 elapsed=5.3s lr=2.000000e-03 loss=0.2442 loss_recon=0.2442 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9450 corrupt_frac=0.7000 acc_corrupt=0.9250 loss_corrupt=0.2442 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9064 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.5454 out_g_norm=1.2266 acc_corrupt_t_0p2_0p4=0.9398 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0013 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 2500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=2050 epoch=1025/1250 epoch_step=2/2 micro_steps=2050 elapsed=6.1s lr=2.000000e-03 loss=0.3897 loss_recon=0.3897 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9475 corrupt_frac=0.7025 acc_corrupt=0.9253 loss_corrupt=0.3897 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8603 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.5831 out_g_norm=1.2381 acc_corrupt_t_0p2_0p4=0.9832 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1091 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=2100 epoch=1050/1250 epoch_step=2/2 micro_steps=2100 elapsed=4.5s lr=2.000000e-03 loss=0.2549 loss_recon=0.2549 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9500 corrupt_frac=0.7075 acc_corrupt=0.9329 loss_corrupt=0.2549 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9235 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.6151 out_g_norm=1.2089 acc_corrupt_t_0p2_0p4=0.9467 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0006 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=2150 epoch=1075/1250 epoch_step=2/2 micro_steps=2150 elapsed=1.5s lr=2.000000e-03 loss=0.2001 loss_recon=0.2001 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9475 corrupt_frac=0.7225 acc_corrupt=0.9377 loss_corrupt=0.2001 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9174 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.6559 out_g_norm=1.3223 acc_corrupt_t_0p0_0p2=0.9419 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.0864 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=2200 epoch=1100/1250 epoch_step=2/2 micro_steps=2200 elapsed=2.0s lr=2.000000e-03 loss=0.0891 loss_recon=0.0891 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.6950 acc_corrupt=0.9676 loss_corrupt=0.0891 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9384 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.6873 out_g_norm=1.2010 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0082 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=2250 epoch=1125/1250 epoch_step=2/2 micro_steps=2250 elapsed=1.5s lr=2.000000e-03 loss=0.0757 loss_recon=0.0757 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.7425 acc_corrupt=0.9697 loss_corrupt=0.0757 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9434 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.7293 out_g_norm=1.0732 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0019 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=2300 epoch=1150/1250 epoch_step=2/2 micro_steps=2300 elapsed=1.5s lr=2.000000e-03 loss=0.1484 loss_recon=0.1484 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1484 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9504 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.7570 out_g_norm=1.1590 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0021 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=2350 epoch=1175/1250 epoch_step=2/2 micro_steps=2350 elapsed=1.5s lr=2.000000e-03 loss=0.4873 loss_recon=0.4873 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9000 corrupt_frac=0.7250 acc_corrupt=0.8759 loss_corrupt=0.4873 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.8299 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.7761 out_g_norm=1.2913 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9559 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0059 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=2400 epoch=1200/1250 epoch_step=2/2 micro_steps=2400 elapsed=1.4s lr=2.000000e-03 loss=0.0507 loss_recon=0.0507 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9823 loss_corrupt=0.0507 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.7961 out_g_norm=0.8929 acc_corrupt_t_0p0_0p2=0.9624 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0089 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=2450 epoch=1225/1250 epoch_step=2/2 micro_steps=2450 elapsed=5.1s lr=2.000000e-03 loss=0.4240 loss_recon=0.4240 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9425 corrupt_frac=0.6800 acc_corrupt=0.9228 loss_corrupt=0.4240 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9020 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.8328 out_g_norm=1.2583 acc_corrupt_t_0p2_0p4=0.9241 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.3923 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=2500 epoch=1250/1250 epoch_step=2/2 micro_steps=2500 elapsed=5.3s lr=2.000000e-03 loss=0.1704 loss_recon=0.1704 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.7000 acc_corrupt=0.9500 loss_corrupt=0.1704 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9357 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.8630 out_g_norm=0.9495 acc_corrupt_t_0p2_0p4=0.9639 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0380 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=2501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=2550 epoch=1275/1500 epoch_step=2/2 micro_steps=2550 elapsed=6.2s lr=2.000000e-03 loss=0.2618 loss_recon=0.2618 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9525 corrupt_frac=0.7025 acc_corrupt=0.9359 loss_corrupt=0.2618 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8750 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.8960 out_g_norm=1.0191 acc_corrupt_t_0p2_0p4=0.9916 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0847 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=2600 epoch=1300/1500 epoch_step=2/2 micro_steps=2600 elapsed=5.5s lr=2.000000e-03 loss=0.1449 loss_recon=0.1449 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9525 corrupt_frac=0.7075 acc_corrupt=0.9470 loss_corrupt=0.1449 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9490 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.9217 out_g_norm=0.9914 acc_corrupt_t_0p2_0p4=0.9333 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0004 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=2650 epoch=1325/1500 epoch_step=2/2 micro_steps=2650 elapsed=5.5s lr=2.000000e-03 loss=0.1596 loss_recon=0.1596 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9575 corrupt_frac=0.7225 acc_corrupt=0.9516 loss_corrupt=0.1596 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.9562 out_g_norm=1.1960 acc_corrupt_t_0p0_0p2=0.9355 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.0227 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=2700 epoch=1350/1500 epoch_step=2/2 micro_steps=2700 elapsed=4.1s lr=2.000000e-03 loss=0.0892 loss_recon=0.0892 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6950 acc_corrupt=0.9640 loss_corrupt=0.0892 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9384 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.9907 out_g_norm=1.0729 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=0.9412 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0570 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=2750 epoch=1375/1500 epoch_step=2/2 micro_steps=2750 elapsed=1.6s lr=2.000000e-03 loss=0.0657 loss_recon=0.0657 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7425 acc_corrupt=0.9697 loss_corrupt=0.0657 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9748 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.0216 out_g_norm=0.8064 acc_corrupt_t_0p2_0p4=0.9590 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0315 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=2800 epoch=1400/1500 epoch_step=2/2 micro_steps=2800 elapsed=2.1s lr=2.000000e-03 loss=0.1628 loss_recon=0.1628 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1628 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9787 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.0525 out_g_norm=0.9450 acc_corrupt_t_0p2_0p4=0.9174 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1005 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=2850 epoch=1425/1500 epoch_step=2/2 micro_steps=2850 elapsed=1.5s lr=2.000000e-03 loss=0.2467 loss_recon=0.2467 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9300 corrupt_frac=0.7250 acc_corrupt=0.9207 loss_corrupt=0.2467 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.8814 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.0716 out_g_norm=1.1778 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.1009 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=2900 epoch=1450/1500 epoch_step=2/2 micro_steps=2900 elapsed=1.5s lr=2.000000e-03 loss=0.0474 loss_recon=0.0474 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7075 acc_corrupt=0.9859 loss_corrupt=0.0474 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9846 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.0913 out_g_norm=0.7563 acc_corrupt_t_0p0_0p2=0.9850 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0163 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=2950 epoch=1475/1500 epoch_step=2/2 micro_steps=2950 elapsed=1.5s lr=2.000000e-03 loss=0.3049 loss_recon=0.3049 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.6800 acc_corrupt=0.9154 loss_corrupt=0.3049 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.8693 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.1194 out_g_norm=1.2396 acc_corrupt_t_0p2_0p4=0.9620 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.9435 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=3000 epoch=1500/1500 epoch_step=2/2 micro_steps=3000 elapsed=3.2s lr=2.000000e-03 loss=0.2097 loss_recon=0.2097 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9575 corrupt_frac=0.7000 acc_corrupt=0.9429 loss_corrupt=0.2097 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9240 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.1468 out_g_norm=0.9625 acc_corrupt_t_0p2_0p4=0.9639 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0005 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 3500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=3050 epoch=1525/1750 epoch_step=2/2 micro_steps=3050 elapsed=2.4s lr=2.000000e-03 loss=0.4195 loss_recon=0.4195 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7025 acc_corrupt=0.9110 loss_corrupt=0.4195 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8456 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.1696 out_g_norm=1.0540 acc_corrupt_t_0p2_0p4=0.9664 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.3355 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=3100 epoch=1550/1750 epoch_step=2/2 micro_steps=3100 elapsed=1.5s lr=2.000000e-03 loss=0.1242 loss_recon=0.1242 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7075 acc_corrupt=0.9611 loss_corrupt=0.1242 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9592 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.1944 out_g_norm=0.7669 acc_corrupt_t_0p2_0p4=0.9600 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=3150 epoch=1575/1750 epoch_step=2/2 micro_steps=3150 elapsed=1.5s lr=2.000000e-03 loss=0.2021 loss_recon=0.2021 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9400 corrupt_frac=0.7225 acc_corrupt=0.9308 loss_corrupt=0.2021 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.8899 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.2316 out_g_norm=1.1105 acc_corrupt_t_0p0_0p2=0.9484 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.2009 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=3200 epoch=1600/1750 epoch_step=2/2 micro_steps=3200 elapsed=1.9s lr=2.000000e-03 loss=0.0387 loss_recon=0.0387 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.6950 acc_corrupt=0.9892 loss_corrupt=0.0387 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9795 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.2584 out_g_norm=0.7571 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0162 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=3250 epoch=1625/1750 epoch_step=2/2 micro_steps=3250 elapsed=5.3s lr=2.000000e-03 loss=0.1816 loss_recon=0.1816 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9450 corrupt_frac=0.7425 acc_corrupt=0.9360 loss_corrupt=0.1816 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9308 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.2863 out_g_norm=0.6708 acc_corrupt_t_0p2_0p4=0.9344 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.4355 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=3300 epoch=1650/1750 epoch_step=2/2 micro_steps=3300 elapsed=5.3s lr=2.000000e-03 loss=0.1433 loss_recon=0.1433 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.6800 acc_corrupt=0.9522 loss_corrupt=0.1433 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9574 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.3141 out_g_norm=0.9053 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0337 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=3350 epoch=1675/1750 epoch_step=2/2 micro_steps=3350 elapsed=5.3s lr=2.000000e-03 loss=0.2891 loss_recon=0.2891 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7250 acc_corrupt=0.9310 loss_corrupt=0.2891 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9072 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.3144 out_g_norm=0.8813 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9706 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0002 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=3400 epoch=1700/1750 epoch_step=2/2 micro_steps=3400 elapsed=3.5s lr=2.000000e-03 loss=0.0401 loss_recon=0.0401 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7075 acc_corrupt=0.9894 loss_corrupt=0.0401 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9923 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.3317 out_g_norm=0.5452 acc_corrupt_t_0p0_0p2=0.9850 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0586 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=3450 epoch=1725/1750 epoch_step=2/2 micro_steps=3450 elapsed=1.5s lr=2.000000e-03 loss=0.1711 loss_recon=0.1711 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9475 corrupt_frac=0.6800 acc_corrupt=0.9338 loss_corrupt=0.1711 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9216 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.3574 out_g_norm=1.0929 acc_corrupt_t_0p2_0p4=0.9241 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1451 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=3500 epoch=1750/1750 epoch_step=2/2 micro_steps=3500 elapsed=2.0s lr=2.000000e-03 loss=0.1374 loss_recon=0.1374 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7000 acc_corrupt=0.9571 loss_corrupt=0.1374 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9357 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.3809 out_g_norm=0.8358 acc_corrupt_t_0p2_0p4=0.9880 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0017 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=3501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=3550 epoch=1775/2000 epoch_step=2/2 micro_steps=3550 elapsed=2.3s lr=2.000000e-03 loss=0.3157 loss_recon=0.3157 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9550 corrupt_frac=0.7025 acc_corrupt=0.9395 loss_corrupt=0.3157 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8897 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.4072 out_g_norm=1.0028 acc_corrupt_t_0p2_0p4=0.9832 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.2006 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=3600 epoch=1800/2000 epoch_step=2/2 micro_steps=3600 elapsed=1.4s lr=2.000000e-03 loss=0.0733 loss_recon=0.0733 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0733 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9745 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.4252 out_g_norm=0.6264 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0004 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=3650 epoch=1825/2000 epoch_step=2/2 micro_steps=3650 elapsed=1.5s lr=2.000000e-03 loss=0.1393 loss_recon=0.1393 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7225 acc_corrupt=0.9550 loss_corrupt=0.1393 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.8991 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.4568 out_g_norm=1.0606 acc_corrupt_t_0p0_0p2=0.9871 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0002 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=3700 epoch=1850/2000 epoch_step=2/2 micro_steps=3700 elapsed=1.5s lr=2.000000e-03 loss=0.1078 loss_recon=0.1078 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6950 acc_corrupt=0.9712 loss_corrupt=0.1078 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9452 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.4780 out_g_norm=0.8177 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=3750 epoch=1875/2000 epoch_step=2/2 micro_steps=3750 elapsed=1.5s lr=2.000000e-03 loss=0.0532 loss_recon=0.0532 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7425 acc_corrupt=0.9798 loss_corrupt=0.0532 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9623 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.4922 out_g_norm=0.6067 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.2035 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=3800 epoch=1900/2000 epoch_step=2/2 micro_steps=3800 elapsed=1.5s lr=2.000000e-03 loss=0.1095 loss_recon=0.1095 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6800 acc_corrupt=0.9632 loss_corrupt=0.1095 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9929 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5144 out_g_norm=0.9959 acc_corrupt_t_0p2_0p4=0.9266 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9444 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0001 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=3850 epoch=1925/2000 epoch_step=2/2 micro_steps=3850 elapsed=1.5s lr=2.000000e-03 loss=0.1444 loss_recon=0.1444 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7250 acc_corrupt=0.9552 loss_corrupt=0.1444 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9381 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5218 out_g_norm=0.8416 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9853 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0008 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=3900 epoch=1950/2000 epoch_step=2/2 micro_steps=3900 elapsed=1.4s lr=2.000000e-03 loss=0.0021 loss_recon=0.0021 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=1.0000 corrupt_frac=0.7075 acc_corrupt=1.0000 loss_corrupt=0.0021 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.5380 out_g_norm=0.5577 acc_corrupt_t_0p0_0p2=1.0000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0034 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=3950 epoch=1975/2000 epoch_step=2/2 micro_steps=3950 elapsed=1.4s lr=2.000000e-03 loss=0.2746 loss_recon=0.2746 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9450 corrupt_frac=0.6800 acc_corrupt=0.9301 loss_corrupt=0.2746 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9216 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5425 out_g_norm=0.9138 acc_corrupt_t_0p2_0p4=0.9114 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0545 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=4000 epoch=2000/2000 epoch_step=2/2 micro_steps=4000 elapsed=1.4s lr=2.000000e-03 loss=0.1321 loss_recon=0.1321 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7000 acc_corrupt=0.9714 loss_corrupt=0.1321 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9766 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5645 out_g_norm=0.7020 acc_corrupt_t_0p2_0p4=0.9518 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 4500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=4050 epoch=2025/2250 epoch_step=2/2 micro_steps=4050 elapsed=2.3s lr=2.000000e-03 loss=0.3990 loss_recon=0.3990 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7025 acc_corrupt=0.9181 loss_corrupt=0.3990 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8382 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5842 out_g_norm=1.0386 acc_corrupt_t_0p2_0p4=0.9916 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1540 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=4100 epoch=2050/2250 epoch_step=2/2 micro_steps=4100 elapsed=1.4s lr=2.000000e-03 loss=0.0506 loss_recon=0.0506 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0506 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9745 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.5922 out_g_norm=0.6904 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=4150 epoch=2075/2250 epoch_step=2/2 micro_steps=4150 elapsed=1.4s lr=2.000000e-03 loss=0.1163 loss_recon=0.1163 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7225 acc_corrupt=0.9619 loss_corrupt=0.1163 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.6061 out_g_norm=0.7886 acc_corrupt_t_0p0_0p2=0.9742 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0032 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=4200 epoch=2100/2250 epoch_step=2/2 micro_steps=4200 elapsed=1.4s lr=2.000000e-03 loss=0.0598 loss_recon=0.0598 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.6950 acc_corrupt=0.9820 loss_corrupt=0.0598 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9726 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.6199 out_g_norm=0.8617 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0002 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=4250 epoch=2125/2250 epoch_step=2/2 micro_steps=4250 elapsed=1.4s lr=2.000000e-03 loss=0.0472 loss_recon=0.0472 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.7425 acc_corrupt=0.9798 loss_corrupt=0.0472 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9748 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.6459 out_g_norm=0.6007 acc_corrupt_t_0p2_0p4=0.9836 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.2080 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=4300 epoch=2150/2250 epoch_step=2/2 micro_steps=4300 elapsed=1.4s lr=2.000000e-03 loss=0.1417 loss_recon=0.1417 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.6800 acc_corrupt=0.9632 loss_corrupt=0.1417 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9787 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.6758 out_g_norm=0.8037 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0001 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=4350 epoch=2175/2250 epoch_step=2/2 micro_steps=4350 elapsed=1.3s lr=2.000000e-03 loss=0.1965 loss_recon=0.1965 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9500 corrupt_frac=0.7250 acc_corrupt=0.9379 loss_corrupt=0.1965 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9072 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.6977 out_g_norm=0.7582 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0045 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=4400 epoch=2200/2250 epoch_step=2/2 micro_steps=4400 elapsed=1.4s lr=2.000000e-03 loss=0.0137 loss_recon=0.0137 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7075 acc_corrupt=0.9965 loss_corrupt=0.0137 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.7119 out_g_norm=0.5357 acc_corrupt_t_0p0_0p2=0.9925 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0007 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=4450 epoch=2225/2250 epoch_step=2/2 micro_steps=4450 elapsed=1.4s lr=2.000000e-03 loss=0.3560 loss_recon=0.3560 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9400 corrupt_frac=0.6800 acc_corrupt=0.9191 loss_corrupt=0.3560 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9150 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7214 out_g_norm=1.1579 acc_corrupt_t_0p2_0p4=0.8861 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.5420 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=4500 epoch=2250/2250 epoch_step=2/2 micro_steps=4500 elapsed=1.4s lr=2.000000e-03 loss=0.0562 loss_recon=0.0562 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7000 acc_corrupt=0.9786 loss_corrupt=0.0562 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9708 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7096 out_g_norm=0.6786 acc_corrupt_t_0p2_0p4=0.9880 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0025 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=4501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=4550 epoch=2275/2500 epoch_step=2/2 micro_steps=4550 elapsed=2.1s lr=2.000000e-03 loss=0.2218 loss_recon=0.2218 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.7025 acc_corrupt=0.9502 loss_corrupt=0.2218 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9044 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7226 out_g_norm=0.6319 acc_corrupt_t_0p2_0p4=0.9916 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1808 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=4600 epoch=2300/2500 epoch_step=2/2 micro_steps=4600 elapsed=1.3s lr=2.000000e-03 loss=0.1411 loss_recon=0.1411 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7075 acc_corrupt=0.9541 loss_corrupt=0.1411 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9643 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7475 out_g_norm=0.6650 acc_corrupt_t_0p2_0p4=0.9200 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=4650 epoch=2325/2500 epoch_step=2/2 micro_steps=4650 elapsed=1.3s lr=2.000000e-03 loss=0.1109 loss_recon=0.1109 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.7225 acc_corrupt=0.9516 loss_corrupt=0.1109 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9725 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.7699 out_g_norm=0.9117 acc_corrupt_t_0p0_0p2=0.9290 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.8158 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=4700 epoch=2350/2500 epoch_step=2/2 micro_steps=4700 elapsed=1.4s lr=2.000000e-03 loss=0.0931 loss_recon=0.0931 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.6950 acc_corrupt=0.9712 loss_corrupt=0.0931 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9521 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7818 out_g_norm=0.7134 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0016 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=4750 epoch=2375/2500 epoch_step=2/2 micro_steps=4750 elapsed=1.4s lr=2.000000e-03 loss=0.0364 loss_recon=0.0364 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7425 acc_corrupt=0.9798 loss_corrupt=0.0364 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9623 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7863 out_g_norm=0.7574 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0291 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=4800 epoch=2400/2500 epoch_step=2/2 micro_steps=4800 elapsed=1.4s lr=2.000000e-03 loss=0.0405 loss_recon=0.0405 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.6800 acc_corrupt=0.9816 loss_corrupt=0.0405 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9929 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.7949 out_g_norm=0.6195 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0005 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=4850 epoch=2425/2500 epoch_step=2/2 micro_steps=4850 elapsed=1.3s lr=2.000000e-03 loss=0.2645 loss_recon=0.2645 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7250 acc_corrupt=0.9310 loss_corrupt=0.2645 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9072 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8139 out_g_norm=0.7404 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9706 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0010 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=4900 epoch=2450/2500 epoch_step=2/2 micro_steps=4900 elapsed=1.3s lr=2.000000e-03 loss=0.0347 loss_recon=0.0347 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0347 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9692 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.8326 out_g_norm=0.6777 acc_corrupt_t_0p0_0p2=0.9850 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0629 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=4950 epoch=2475/2500 epoch_step=2/2 micro_steps=4950 elapsed=1.3s lr=2.000000e-03 loss=0.1796 loss_recon=0.1796 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.6800 acc_corrupt=0.9632 loss_corrupt=0.1796 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9477 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8466 out_g_norm=0.8300 acc_corrupt_t_0p2_0p4=0.9747 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1380 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=5000 epoch=2500/2500 epoch_step=2/2 micro_steps=5000 elapsed=1.3s lr=2.000000e-03 loss=0.1036 loss_recon=0.1036 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7000 acc_corrupt=0.9786 loss_corrupt=0.1036 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9942 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8536 out_g_norm=0.5665 acc_corrupt_t_0p2_0p4=0.9398 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0003 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 5500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=5050 epoch=2525/2750 epoch_step=2/2 micro_steps=5050 elapsed=2.2s lr=2.000000e-03 loss=0.1116 loss_recon=0.1116 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7025 acc_corrupt=0.9680 loss_corrupt=0.1116 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9338 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8720 out_g_norm=0.7236 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0049 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=5100 epoch=2550/2750 epoch_step=2/2 micro_steps=5100 elapsed=1.4s lr=2.000000e-03 loss=0.1306 loss_recon=0.1306 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.7075 acc_corrupt=0.9611 loss_corrupt=0.1306 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9643 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8857 out_g_norm=0.7381 acc_corrupt_t_0p2_0p4=0.9467 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=5150 epoch=2575/2750 epoch_step=2/2 micro_steps=5150 elapsed=1.4s lr=2.000000e-03 loss=0.2883 loss_recon=0.2883 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9525 corrupt_frac=0.7225 acc_corrupt=0.9412 loss_corrupt=0.2883 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9450 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.9029 out_g_norm=1.0097 acc_corrupt_t_0p0_0p2=0.9290 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0047 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=5200 epoch=2600/2750 epoch_step=2/2 micro_steps=5200 elapsed=1.4s lr=2.000000e-03 loss=0.0264 loss_recon=0.0264 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9875 corrupt_frac=0.6950 acc_corrupt=0.9892 loss_corrupt=0.0264 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9795 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8914 out_g_norm=0.5979 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0002 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=5250 epoch=2625/2750 epoch_step=2/2 micro_steps=5250 elapsed=1.4s lr=2.000000e-03 loss=0.1433 loss_recon=0.1433 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.7425 acc_corrupt=0.9630 loss_corrupt=0.1433 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9497 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.9007 out_g_norm=0.5338 acc_corrupt_t_0p2_0p4=0.9754 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0060 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=5300 epoch=2650/2750 epoch_step=2/2 micro_steps=5300 elapsed=1.4s lr=2.000000e-03 loss=0.0480 loss_recon=0.0480 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.6800 acc_corrupt=0.9853 loss_corrupt=0.0480 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9858 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.9311 out_g_norm=0.6721 acc_corrupt_t_0p2_0p4=0.9817 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0019 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=5350 epoch=2675/2750 epoch_step=2/2 micro_steps=5350 elapsed=1.5s lr=2.000000e-03 loss=0.1122 loss_recon=0.1122 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.7250 acc_corrupt=0.9655 loss_corrupt=0.1122 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9485 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.9510 out_g_norm=0.7453 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0021 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=5400 epoch=2700/2750 epoch_step=2/2 micro_steps=5400 elapsed=1.5s lr=2.000000e-03 loss=0.0134 loss_recon=0.0134 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7075 acc_corrupt=0.9965 loss_corrupt=0.0134 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.9822 out_g_norm=0.4586 acc_corrupt_t_0p0_0p2=0.9925 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0073 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=5450 epoch=2725/2750 epoch_step=2/2 micro_steps=5450 elapsed=1.4s lr=2.000000e-03 loss=0.2420 loss_recon=0.2420 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9500 corrupt_frac=0.6800 acc_corrupt=0.9338 loss_corrupt=0.2420 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9150 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0115 out_g_norm=0.8363 acc_corrupt_t_0p2_0p4=0.9367 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.4631 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=5500 epoch=2750/2750 epoch_step=2/2 micro_steps=5500 elapsed=1.4s lr=2.000000e-03 loss=0.0219 loss_recon=0.0219 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.7000 acc_corrupt=0.9893 loss_corrupt=0.0219 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9883 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0169 out_g_norm=0.6515 acc_corrupt_t_0p2_0p4=0.9880 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=5501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=5550 epoch=2775/3000 epoch_step=2/2 micro_steps=5550 elapsed=2.4s lr=2.000000e-03 loss=0.2863 loss_recon=0.2863 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.7025 acc_corrupt=0.9609 loss_corrupt=0.2863 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9338 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0411 out_g_norm=0.7279 acc_corrupt_t_0p2_0p4=0.9832 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1612 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=5600 epoch=2800/3000 epoch_step=2/2 micro_steps=5600 elapsed=1.4s lr=2.000000e-03 loss=0.1565 loss_recon=0.1565 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7075 acc_corrupt=0.9682 loss_corrupt=0.1565 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9541 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0471 out_g_norm=0.5814 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=5650 epoch=2825/3000 epoch_step=2/2 micro_steps=5650 elapsed=1.4s lr=2.000000e-03 loss=0.1181 loss_recon=0.1181 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7225 acc_corrupt=0.9654 loss_corrupt=0.1181 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.0731 out_g_norm=0.8338 acc_corrupt_t_0p0_0p2=0.9806 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0251 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=5700 epoch=2850/3000 epoch_step=2/2 micro_steps=5700 elapsed=1.5s lr=2.000000e-03 loss=0.1398 loss_recon=0.1398 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6950 acc_corrupt=0.9640 loss_corrupt=0.1398 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9384 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0906 out_g_norm=0.8427 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=0.9412 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0065 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=5750 epoch=2875/3000 epoch_step=2/2 micro_steps=5750 elapsed=1.4s lr=2.000000e-03 loss=0.0895 loss_recon=0.0895 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7425 acc_corrupt=0.9731 loss_corrupt=0.0895 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9748 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.0890 out_g_norm=0.4425 acc_corrupt_t_0p2_0p4=0.9672 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0218 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=5800 epoch=2900/3000 epoch_step=2/2 micro_steps=5800 elapsed=1.4s lr=2.000000e-03 loss=0.2101 loss_recon=0.2101 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6800 acc_corrupt=0.9522 loss_corrupt=0.2101 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9433 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1165 out_g_norm=0.6883 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0385 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=5850 epoch=2925/3000 epoch_step=2/2 micro_steps=5850 elapsed=1.4s lr=2.000000e-03 loss=0.2296 loss_recon=0.2296 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9525 corrupt_frac=0.7250 acc_corrupt=0.9414 loss_corrupt=0.2296 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9124 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1266 out_g_norm=0.6380 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0004 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=5900 epoch=2950/3000 epoch_step=2/2 micro_steps=5900 elapsed=1.4s lr=2.000000e-03 loss=0.0680 loss_recon=0.0680 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0680 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.1201 out_g_norm=0.5455 acc_corrupt_t_0p0_0p2=0.9549 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0023 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=5950 epoch=2975/3000 epoch_step=2/2 micro_steps=5950 elapsed=1.4s lr=2.000000e-03 loss=0.1330 loss_recon=0.1330 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.6800 acc_corrupt=0.9706 loss_corrupt=0.1330 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9477 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1193 out_g_norm=0.7107 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0014 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=6000 epoch=3000/3000 epoch_step=2/2 micro_steps=6000 elapsed=1.4s lr=2.000000e-03 loss=0.0783 loss_recon=0.0783 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7000 acc_corrupt=0.9786 loss_corrupt=0.0783 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9649 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1209 out_g_norm=0.6830 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 6500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=6050 epoch=3025/3250 epoch_step=2/2 micro_steps=6050 elapsed=2.4s lr=2.000000e-03 loss=0.0740 loss_recon=0.0740 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7025 acc_corrupt=0.9680 loss_corrupt=0.0740 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9338 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1093 out_g_norm=0.8547 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1229 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=6100 epoch=3050/3250 epoch_step=2/2 micro_steps=6100 elapsed=1.5s lr=2.000000e-03 loss=0.0418 loss_recon=0.0418 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7075 acc_corrupt=0.9859 loss_corrupt=0.0418 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9949 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1063 out_g_norm=0.5673 acc_corrupt_t_0p2_0p4=0.9600 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=6150 epoch=3075/3250 epoch_step=2/2 micro_steps=6150 elapsed=1.5s lr=2.000000e-03 loss=0.1203 loss_recon=0.1203 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7225 acc_corrupt=0.9723 loss_corrupt=0.1203 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.1107 out_g_norm=0.7245 acc_corrupt_t_0p0_0p2=0.9484 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.6976 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=6200 epoch=3100/3250 epoch_step=2/2 micro_steps=6200 elapsed=1.5s lr=2.000000e-03 loss=0.0674 loss_recon=0.0674 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.6950 acc_corrupt=0.9784 loss_corrupt=0.0674 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9589 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1138 out_g_norm=0.5856 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=6250 epoch=3125/3250 epoch_step=2/2 micro_steps=6250 elapsed=1.5s lr=2.000000e-03 loss=0.0791 loss_recon=0.0791 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7425 acc_corrupt=0.9764 loss_corrupt=0.0791 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9623 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1270 out_g_norm=0.5663 acc_corrupt_t_0p2_0p4=0.9918 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.3383 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=6300 epoch=3150/3250 epoch_step=2/2 micro_steps=6300 elapsed=1.5s lr=2.000000e-03 loss=0.1812 loss_recon=0.1812 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.6800 acc_corrupt=0.9596 loss_corrupt=0.1812 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9574 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1321 out_g_norm=0.8312 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.9200 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=6350 epoch=3175/3250 epoch_step=2/2 micro_steps=6350 elapsed=1.5s lr=2.000000e-03 loss=0.0873 loss_recon=0.0873 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.7250 acc_corrupt=0.9655 loss_corrupt=0.0873 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9485 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1182 out_g_norm=0.7125 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0018 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=6400 epoch=3200/3250 epoch_step=2/2 micro_steps=6400 elapsed=1.5s lr=2.000000e-03 loss=0.0115 loss_recon=0.0115 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7075 acc_corrupt=1.0000 loss_corrupt=0.0115 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.1233 out_g_norm=0.5552 acc_corrupt_t_0p0_0p2=1.0000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1565 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=6450 epoch=3225/3250 epoch_step=2/2 micro_steps=6450 elapsed=1.5s lr=2.000000e-03 loss=0.1603 loss_recon=0.1603 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1603 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9346 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1138 out_g_norm=0.6455 acc_corrupt_t_0p2_0p4=0.9747 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.5827 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=6500 epoch=3250/3250 epoch_step=2/2 micro_steps=6500 elapsed=1.5s lr=2.000000e-03 loss=0.0231 loss_recon=0.0231 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7000 acc_corrupt=0.9929 loss_corrupt=0.0231 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9883 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1286 out_g_norm=0.5590 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=6501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=6550 epoch=3275/3500 epoch_step=2/2 micro_steps=6550 elapsed=2.3s lr=2.000000e-03 loss=0.3264 loss_recon=0.3264 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9450 corrupt_frac=0.7025 acc_corrupt=0.9253 loss_corrupt=0.3264 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8750 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1534 out_g_norm=0.9309 acc_corrupt_t_0p2_0p4=0.9664 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.9883 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=6600 epoch=3300/3500 epoch_step=2/2 micro_steps=6600 elapsed=1.4s lr=2.000000e-03 loss=0.1838 loss_recon=0.1838 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9500 corrupt_frac=0.7075 acc_corrupt=0.9399 loss_corrupt=0.1838 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9133 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1470 out_g_norm=0.6059 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=6650 epoch=3325/3500 epoch_step=2/2 micro_steps=6650 elapsed=1.4s lr=2.000000e-03 loss=0.0714 loss_recon=0.0714 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7225 acc_corrupt=0.9723 loss_corrupt=0.0714 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.1518 out_g_norm=0.7894 acc_corrupt_t_0p0_0p2=0.9742 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0053 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=6700 epoch=3350/3500 epoch_step=2/2 micro_steps=6700 elapsed=1.4s lr=2.000000e-03 loss=0.0505 loss_recon=0.0505 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.6950 acc_corrupt=0.9820 loss_corrupt=0.0505 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9726 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1481 out_g_norm=0.5320 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=6750 epoch=3375/3500 epoch_step=2/2 micro_steps=6750 elapsed=1.4s lr=2.000000e-03 loss=0.0314 loss_recon=0.0314 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.7425 acc_corrupt=0.9899 loss_corrupt=0.0314 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9811 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1683 out_g_norm=0.4526 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0063 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=6800 epoch=3400/3500 epoch_step=2/2 micro_steps=6800 elapsed=1.4s lr=2.000000e-03 loss=0.0154 loss_recon=0.0154 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.6800 acc_corrupt=0.9963 loss_corrupt=0.0154 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=1.0000 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2048 out_g_norm=0.4612 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0301 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=6850 epoch=3425/3500 epoch_step=2/2 micro_steps=6850 elapsed=1.4s lr=2.000000e-03 loss=0.0951 loss_recon=0.0951 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7250 acc_corrupt=0.9759 loss_corrupt=0.0951 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9691 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2176 out_g_norm=0.7125 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9853 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0004 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=6900 epoch=3450/3500 epoch_step=2/2 micro_steps=6900 elapsed=1.4s lr=2.000000e-03 loss=0.0331 loss_recon=0.0331 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7075 acc_corrupt=0.9894 loss_corrupt=0.0331 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9923 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.2086 out_g_norm=0.6309 acc_corrupt_t_0p0_0p2=0.9850 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1320 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=6950 epoch=3475/3500 epoch_step=2/2 micro_steps=6950 elapsed=1.4s lr=2.000000e-03 loss=0.2303 loss_recon=0.2303 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.6800 acc_corrupt=0.9522 loss_corrupt=0.2303 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9281 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2007 out_g_norm=0.9088 acc_corrupt_t_0p2_0p4=0.9747 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.5060 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=7000 epoch=3500/3500 epoch_step=2/2 micro_steps=7000 elapsed=1.4s lr=2.000000e-03 loss=0.0849 loss_recon=0.0849 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.7000 acc_corrupt=0.9714 loss_corrupt=0.0849 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9649 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.1924 out_g_norm=0.6911 acc_corrupt_t_0p2_0p4=0.9759 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=7001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 7500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=7050 epoch=3525/3750 epoch_step=2/2 micro_steps=7050 elapsed=2.2s lr=2.000000e-03 loss=0.1983 loss_recon=0.1983 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7025 acc_corrupt=0.9502 loss_corrupt=0.1983 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9191 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2034 out_g_norm=0.7669 acc_corrupt_t_0p2_0p4=0.9748 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1131 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=7100 epoch=3550/3750 epoch_step=2/2 micro_steps=7100 elapsed=1.3s lr=2.000000e-03 loss=0.0938 loss_recon=0.0938 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0938 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9745 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2040 out_g_norm=0.4341 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=7150 epoch=3575/3750 epoch_step=2/2 micro_steps=7150 elapsed=1.3s lr=2.000000e-03 loss=0.3311 loss_recon=0.3311 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9500 corrupt_frac=0.7225 acc_corrupt=0.9377 loss_corrupt=0.3311 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.8991 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.2192 out_g_norm=0.7305 acc_corrupt_t_0p0_0p2=0.9548 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.8262 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=7200 epoch=3600/3750 epoch_step=2/2 micro_steps=7200 elapsed=1.3s lr=2.000000e-03 loss=0.0208 loss_recon=0.0208 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.6950 acc_corrupt=0.9928 loss_corrupt=0.0208 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9863 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2382 out_g_norm=0.4983 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0006 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=7250 epoch=3625/3750 epoch_step=2/2 micro_steps=7250 elapsed=1.3s lr=2.000000e-03 loss=0.0555 loss_recon=0.0555 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9875 corrupt_frac=0.7425 acc_corrupt=0.9832 loss_corrupt=0.0555 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9811 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2589 out_g_norm=0.4774 acc_corrupt_t_0p2_0p4=0.9836 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0410 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=7300 epoch=3650/3750 epoch_step=2/2 micro_steps=7300 elapsed=1.3s lr=2.000000e-03 loss=0.2075 loss_recon=0.2075 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.6800 acc_corrupt=0.9596 loss_corrupt=0.2075 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9787 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2679 out_g_norm=0.5488 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9444 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0009 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=7350 epoch=3675/3750 epoch_step=2/2 micro_steps=7350 elapsed=1.3s lr=2.000000e-03 loss=0.0789 loss_recon=0.0789 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7250 acc_corrupt=0.9724 loss_corrupt=0.0789 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9588 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2890 out_g_norm=0.4187 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0008 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=7400 epoch=3700/3750 epoch_step=2/2 micro_steps=7400 elapsed=1.4s lr=2.000000e-03 loss=0.0416 loss_recon=0.0416 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9894 loss_corrupt=0.0416 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3071 out_g_norm=0.5139 acc_corrupt_t_0p0_0p2=0.9774 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0368 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=7450 epoch=3725/3750 epoch_step=2/2 micro_steps=7450 elapsed=1.3s lr=2.000000e-03 loss=0.1113 loss_recon=0.1113 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9650 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1113 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9412 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3037 out_g_norm=0.8474 acc_corrupt_t_0p2_0p4=0.9620 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0131 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=7500 epoch=3750/3750 epoch_step=2/2 micro_steps=7500 elapsed=1.3s lr=2.000000e-03 loss=0.1483 loss_recon=0.1483 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7000 acc_corrupt=0.9643 loss_corrupt=0.1483 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9649 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2986 out_g_norm=0.6401 acc_corrupt_t_0p2_0p4=0.9518 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0003 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=7501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 8000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=7550 epoch=3775/4000 epoch_step=2/2 micro_steps=7550 elapsed=2.2s lr=2.000000e-03 loss=0.2130 loss_recon=0.2130 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.7025 acc_corrupt=0.9466 loss_corrupt=0.2130 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9191 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3113 out_g_norm=0.7629 acc_corrupt_t_0p2_0p4=0.9664 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.3391 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=7600 epoch=3800/4000 epoch_step=2/2 micro_steps=7600 elapsed=1.4s lr=2.000000e-03 loss=0.0546 loss_recon=0.0546 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.0546 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9745 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3270 out_g_norm=0.6343 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0003 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=7650 epoch=3825/4000 epoch_step=2/2 micro_steps=7650 elapsed=1.4s lr=2.000000e-03 loss=0.1286 loss_recon=0.1286 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7225 acc_corrupt=0.9585 loss_corrupt=0.1286 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3311 out_g_norm=0.9253 acc_corrupt_t_0p0_0p2=0.9548 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0271 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=7700 epoch=3850/4000 epoch_step=2/2 micro_steps=7700 elapsed=1.4s lr=2.000000e-03 loss=0.1087 loss_recon=0.1087 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9725 corrupt_frac=0.6950 acc_corrupt=0.9676 loss_corrupt=0.1087 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9384 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3217 out_g_norm=0.7887 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0003 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=7750 epoch=3875/4000 epoch_step=2/2 micro_steps=7750 elapsed=1.4s lr=2.000000e-03 loss=0.0774 loss_recon=0.0774 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7425 acc_corrupt=0.9798 loss_corrupt=0.0774 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9874 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2993 out_g_norm=0.7384 acc_corrupt_t_0p2_0p4=0.9672 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1282 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=7800 epoch=3900/4000 epoch_step=2/2 micro_steps=7800 elapsed=1.4s lr=2.000000e-03 loss=0.1334 loss_recon=0.1334 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.6800 acc_corrupt=0.9632 loss_corrupt=0.1334 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9716 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3093 out_g_norm=0.7299 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9444 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0134 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=7850 epoch=3925/4000 epoch_step=2/2 micro_steps=7850 elapsed=1.4s lr=2.000000e-03 loss=0.0290 loss_recon=0.0290 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7250 acc_corrupt=0.9862 loss_corrupt=0.0290 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9794 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3206 out_g_norm=0.4569 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0006 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=7900 epoch=3950/4000 epoch_step=2/2 micro_steps=7900 elapsed=1.4s lr=2.000000e-03 loss=0.0258 loss_recon=0.0258 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9823 loss_corrupt=0.0258 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9923 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3252 out_g_norm=0.5253 acc_corrupt_t_0p0_0p2=0.9699 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=7950 epoch=3975/4000 epoch_step=2/2 micro_steps=7950 elapsed=1.4s lr=2.000000e-03 loss=0.1596 loss_recon=0.1596 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.6800 acc_corrupt=0.9596 loss_corrupt=0.1596 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9281 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3189 out_g_norm=0.6392 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=1.2082 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=8000 epoch=4000/4000 epoch_step=2/2 micro_steps=8000 elapsed=1.4s lr=2.000000e-03 loss=0.0197 loss_recon=0.0197 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.7000 acc_corrupt=0.9893 loss_corrupt=0.0197 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9825 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3223 out_g_norm=0.5437 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0029 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=8001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 8500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=8050 epoch=4025/4250 epoch_step=2/2 micro_steps=8050 elapsed=2.4s lr=2.000000e-03 loss=0.0637 loss_recon=0.0637 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9875 corrupt_frac=0.7025 acc_corrupt=0.9822 loss_corrupt=0.0637 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9632 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3225 out_g_norm=0.6663 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0002 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=8100 epoch=4050/4250 epoch_step=2/2 micro_steps=8100 elapsed=1.5s lr=2.000000e-03 loss=0.0813 loss_recon=0.0813 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.7075 acc_corrupt=0.9647 loss_corrupt=0.0813 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9541 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3234 out_g_norm=0.5989 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=8150 epoch=4075/4250 epoch_step=2/2 micro_steps=8150 elapsed=1.4s lr=2.000000e-03 loss=0.0420 loss_recon=0.0420 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.7225 acc_corrupt=0.9792 loss_corrupt=0.0420 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9725 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3216 out_g_norm=0.8068 acc_corrupt_t_0p0_0p2=0.9806 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0179 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=8200 epoch=4100/4250 epoch_step=2/2 micro_steps=8200 elapsed=1.4s lr=2.000000e-03 loss=0.0416 loss_recon=0.0416 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.6950 acc_corrupt=0.9892 loss_corrupt=0.0416 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9932 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3214 out_g_norm=0.5951 acc_corrupt_t_0p2_0p4=0.9817 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0159 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=8250 epoch=4125/4250 epoch_step=2/2 micro_steps=8250 elapsed=1.4s lr=2.000000e-03 loss=0.0303 loss_recon=0.0303 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.7425 acc_corrupt=0.9899 loss_corrupt=0.0303 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9811 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3326 out_g_norm=0.3535 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0038 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=8300 epoch=4150/4250 epoch_step=2/2 micro_steps=8300 elapsed=1.4s lr=2.000000e-03 loss=0.1504 loss_recon=0.1504 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1504 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9504 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3596 out_g_norm=0.5020 acc_corrupt_t_0p2_0p4=0.9541 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.6900 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=8350 epoch=4175/4250 epoch_step=2/2 micro_steps=8350 elapsed=1.4s lr=2.000000e-03 loss=0.0514 loss_recon=0.0514 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9800 corrupt_frac=0.7250 acc_corrupt=0.9759 loss_corrupt=0.0514 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9639 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3581 out_g_norm=0.7910 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0022 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=8400 epoch=4200/4250 epoch_step=2/2 micro_steps=8400 elapsed=1.4s lr=2.000000e-03 loss=0.0469 loss_recon=0.0469 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7075 acc_corrupt=0.9859 loss_corrupt=0.0469 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=0.9923 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3297 out_g_norm=0.6176 acc_corrupt_t_0p0_0p2=0.9774 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0006 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=8450 epoch=4225/4250 epoch_step=2/2 micro_steps=8450 elapsed=1.4s lr=2.000000e-03 loss=0.1647 loss_recon=0.1647 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6800 acc_corrupt=0.9596 loss_corrupt=0.1647 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9281 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3186 out_g_norm=0.6007 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.4167 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=8500 epoch=4250/4250 epoch_step=2/2 micro_steps=8500 elapsed=1.4s lr=2.000000e-03 loss=0.0659 loss_recon=0.0659 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7000 acc_corrupt=0.9714 loss_corrupt=0.0659 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9766 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3313 out_g_norm=0.6185 acc_corrupt_t_0p2_0p4=0.9518 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0001 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=8501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 9000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=8550 epoch=4275/4500 epoch_step=2/2 micro_steps=8550 elapsed=2.3s lr=2.000000e-03 loss=0.1441 loss_recon=0.1441 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.7025 acc_corrupt=0.9573 loss_corrupt=0.1441 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9559 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3249 out_g_norm=0.7396 acc_corrupt_t_0p2_0p4=0.9496 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=2.7761 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=8600 epoch=4300/4500 epoch_step=2/2 micro_steps=8600 elapsed=1.4s lr=2.000000e-03 loss=0.0501 loss_recon=0.0501 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7075 acc_corrupt=0.9894 loss_corrupt=0.0501 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9898 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3398 out_g_norm=0.4813 acc_corrupt_t_0p2_0p4=0.9867 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=8650 epoch=4325/4500 epoch_step=2/2 micro_steps=8650 elapsed=1.4s lr=2.000000e-03 loss=0.0727 loss_recon=0.0727 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7225 acc_corrupt=0.9792 loss_corrupt=0.0727 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9817 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3560 out_g_norm=0.8159 acc_corrupt_t_0p0_0p2=0.9742 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0748 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=8700 epoch=4350/4500 epoch_step=2/2 micro_steps=8700 elapsed=1.4s lr=2.000000e-03 loss=0.0073 loss_recon=0.0073 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.6950 acc_corrupt=0.9964 loss_corrupt=0.0073 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9932 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3707 out_g_norm=0.2916 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=8750 epoch=4375/4500 epoch_step=2/2 micro_steps=8750 elapsed=1.4s lr=2.000000e-03 loss=0.1619 loss_recon=0.1619 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9575 corrupt_frac=0.7425 acc_corrupt=0.9495 loss_corrupt=0.1619 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9182 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3733 out_g_norm=0.6634 acc_corrupt_t_0p2_0p4=0.9836 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0011 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=8800 epoch=4400/4500 epoch_step=2/2 micro_steps=8800 elapsed=1.4s lr=2.000000e-03 loss=0.1809 loss_recon=0.1809 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1809 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9645 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3780 out_g_norm=0.8200 acc_corrupt_t_0p2_0p4=0.9358 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.1273 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=8850 epoch=4425/4500 epoch_step=2/2 micro_steps=8850 elapsed=1.4s lr=2.000000e-03 loss=0.3269 loss_recon=0.3269 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7250 acc_corrupt=0.9241 loss_corrupt=0.3269 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.8969 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3667 out_g_norm=0.8264 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9706 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0029 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=8900 epoch=4450/4500 epoch_step=2/2 micro_steps=8900 elapsed=1.4s lr=2.000000e-03 loss=0.0313 loss_recon=0.0313 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7075 acc_corrupt=0.9965 loss_corrupt=0.0313 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3520 out_g_norm=0.4344 acc_corrupt_t_0p0_0p2=0.9925 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0004 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=8950 epoch=4475/4500 epoch_step=2/2 micro_steps=8950 elapsed=1.4s lr=2.000000e-03 loss=0.3277 loss_recon=0.3277 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9425 corrupt_frac=0.6800 acc_corrupt=0.9301 loss_corrupt=0.3277 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.8889 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3462 out_g_norm=0.6776 acc_corrupt_t_0p2_0p4=0.9873 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9722 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0023 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=9000 epoch=4500/4500 epoch_step=2/2 micro_steps=9000 elapsed=1.4s lr=2.000000e-03 loss=0.0335 loss_recon=0.0335 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7000 acc_corrupt=0.9857 loss_corrupt=0.0335 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9883 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3654 out_g_norm=0.6669 acc_corrupt_t_0p2_0p4=0.9759 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0012 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=9001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 9500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=9050 epoch=4525/4750 epoch_step=2/2 micro_steps=9050 elapsed=2.3s lr=2.000000e-03 loss=0.4423 loss_recon=0.4423 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9550 corrupt_frac=0.7025 acc_corrupt=0.9359 loss_corrupt=0.4423 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.8676 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3667 out_g_norm=0.7765 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0796 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=9100 epoch=4550/4750 epoch_step=2/2 micro_steps=9100 elapsed=1.4s lr=2.000000e-03 loss=0.1148 loss_recon=0.1148 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.7075 acc_corrupt=0.9788 loss_corrupt=0.1148 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9796 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3638 out_g_norm=0.6496 acc_corrupt_t_0p2_0p4=0.9733 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=9150 epoch=4575/4750 epoch_step=2/2 micro_steps=9150 elapsed=1.4s lr=2.000000e-03 loss=0.0377 loss_recon=0.0377 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7225 acc_corrupt=0.9896 loss_corrupt=0.0377 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9908 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3592 out_g_norm=0.7403 acc_corrupt_t_0p0_0p2=0.9871 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.3680 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=9200 epoch=4600/4750 epoch_step=2/2 micro_steps=9200 elapsed=1.4s lr=2.000000e-03 loss=0.0880 loss_recon=0.0880 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.6950 acc_corrupt=0.9640 loss_corrupt=0.0880 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9315 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3503 out_g_norm=0.6916 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=9250 epoch=4625/4750 epoch_step=2/2 micro_steps=9250 elapsed=1.4s lr=2.000000e-03 loss=0.0310 loss_recon=0.0310 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7425 acc_corrupt=0.9899 loss_corrupt=0.0310 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9937 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3438 out_g_norm=0.4198 acc_corrupt_t_0p2_0p4=0.9836 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.1550 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=9300 epoch=4650/4750 epoch_step=2/2 micro_steps=9300 elapsed=1.4s lr=2.000000e-03 loss=0.0808 loss_recon=0.0808 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.6800 acc_corrupt=0.9743 loss_corrupt=0.0808 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9787 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3593 out_g_norm=0.5964 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0006 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=9350 epoch=4675/4750 epoch_step=2/2 micro_steps=9350 elapsed=1.4s lr=2.000000e-03 loss=0.0903 loss_recon=0.0903 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.7250 acc_corrupt=0.9655 loss_corrupt=0.0903 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.9588 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3514 out_g_norm=0.5586 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9706 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0013 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=9400 epoch=4700/4750 epoch_step=2/2 micro_steps=9400 elapsed=1.4s lr=2.000000e-03 loss=0.0011 loss_recon=0.0011 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=1.0000 corrupt_frac=0.7075 acc_corrupt=1.0000 loss_corrupt=0.0011 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.3663 out_g_norm=0.3397 acc_corrupt_t_0p0_0p2=1.0000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0002 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=9450 epoch=4725/4750 epoch_step=2/2 micro_steps=9450 elapsed=1.4s lr=2.000000e-03 loss=0.1517 loss_recon=0.1517 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9675 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1517 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9346 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3881 out_g_norm=0.7953 acc_corrupt_t_0p2_0p4=0.9747 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.3198 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=9500 epoch=4750/4750 epoch_step=2/2 micro_steps=9500 elapsed=1.4s lr=2.000000e-03 loss=0.1483 loss_recon=0.1483 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9775 corrupt_frac=0.7000 acc_corrupt=0.9679 loss_corrupt=0.1483 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9532 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3866 out_g_norm=0.6274 acc_corrupt_t_0p2_0p4=0.9880 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0007 init_gold_top10=0.3333 init_gold_top100=1.0000 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt start_step=9501 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 47, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_compactv47_3l_hard_ce_onehot", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 10000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1611392, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_compactv47_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "runs/train8_n8_compactv47_3l_hard_ce_onehot/latest.pt" +} +step=9550 epoch=4775/5000 epoch_step=2/2 micro_steps=9550 elapsed=2.2s lr=2.000000e-03 loss=0.1063 loss_recon=0.1063 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9750 corrupt_frac=0.7025 acc_corrupt=0.9644 loss_corrupt=0.1063 wrong_frac=0.8221 init_acc_corrupt=0.1210 acc_corrupt_t_0p0_0p2=0.9706 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3945 out_g_norm=0.5932 acc_corrupt_t_0p2_0p4=0.9496 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.0755 init_gold_top10=0.4286 init_gold_top100=1.0000 +step=9600 epoch=4800/5000 epoch_step=2/2 micro_steps=9600 elapsed=1.3s lr=2.000000e-03 loss=0.0110 loss_recon=0.0110 loss_meanflow=0.0000 mean_model_t=0.1767 mean_corrupt_t=0.1767 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9925 corrupt_frac=0.7075 acc_corrupt=0.9965 loss_corrupt=0.0110 wrong_frac=0.8163 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.9949 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.4086 out_g_norm=0.4750 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0004 init_gold_top10=0.5000 init_gold_top100=1.0000 +step=9650 epoch=4825/5000 epoch_step=2/2 micro_steps=9650 elapsed=1.3s lr=2.000000e-03 loss=0.0372 loss_recon=0.0372 loss_meanflow=0.0000 mean_model_t=0.1988 mean_corrupt_t=0.1988 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9900 corrupt_frac=0.7225 acc_corrupt=0.9862 loss_corrupt=0.0372 wrong_frac=0.7820 init_acc_corrupt=0.1488 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.4109 out_g_norm=0.7889 acc_corrupt_t_0p0_0p2=1.0000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0004 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=9700 epoch=4850/5000 epoch_step=2/2 micro_steps=9700 elapsed=1.3s lr=2.000000e-03 loss=0.1220 loss_recon=0.1220 loss_meanflow=0.0000 mean_model_t=0.2130 mean_corrupt_t=0.2130 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9825 corrupt_frac=0.6950 acc_corrupt=0.9784 loss_corrupt=0.1220 wrong_frac=0.7914 init_acc_corrupt=0.1223 acc_corrupt_t_0p0_0p2=0.9589 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3967 out_g_norm=0.8317 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.2500 init_gold_top100=1.0000 +step=9750 epoch=4875/5000 epoch_step=2/2 micro_steps=9750 elapsed=1.3s lr=2.000000e-03 loss=0.0121 loss_recon=0.0121 loss_meanflow=0.0000 mean_model_t=0.1940 mean_corrupt_t=0.1940 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9950 corrupt_frac=0.7425 acc_corrupt=0.9933 loss_corrupt=0.0121 wrong_frac=0.8249 init_acc_corrupt=0.1313 acc_corrupt_t_0p0_0p2=0.9937 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3838 out_g_norm=0.6158 acc_corrupt_t_0p2_0p4=0.9918 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.3333 init_gold_top100=1.0000 +step=9800 epoch=4900/5000 epoch_step=2/2 micro_steps=9800 elapsed=1.3s lr=2.000000e-03 loss=0.1777 loss_recon=0.1777 loss_meanflow=0.0000 mean_model_t=0.2256 mean_corrupt_t=0.2256 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9700 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1777 wrong_frac=0.7978 init_acc_corrupt=0.1324 acc_corrupt_t_0p0_0p2=0.9362 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.3961 out_g_norm=0.5563 acc_corrupt_t_0p2_0p4=0.9725 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.2448 init_gold_top10=0.2857 init_gold_top100=1.0000 +step=9850 epoch=4925/5000 epoch_step=2/2 micro_steps=9850 elapsed=1.3s lr=2.000000e-03 loss=0.2424 loss_recon=0.2424 loss_meanflow=0.0000 mean_model_t=0.1847 mean_corrupt_t=0.1847 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.7250 acc_corrupt=0.9241 loss_corrupt=0.2424 wrong_frac=0.8069 init_acc_corrupt=0.1241 acc_corrupt_t_0p0_0p2=0.8866 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.4045 out_g_norm=0.8922 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=0.0000 init_gold_top10=0.1667 init_gold_top100=1.0000 +step=9900 epoch=4950/5000 epoch_step=2/2 micro_steps=9900 elapsed=1.3s lr=2.000000e-03 loss=0.0443 loss_recon=0.0443 loss_meanflow=0.0000 mean_model_t=0.2183 mean_corrupt_t=0.2183 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9875 corrupt_frac=0.7075 acc_corrupt=0.9859 loss_corrupt=0.0443 wrong_frac=0.7597 init_acc_corrupt=0.1625 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.4073 out_g_norm=0.3696 acc_corrupt_t_0p0_0p2=0.9699 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.7143 init_gold_top100=1.0000 +step=9950 epoch=4975/5000 epoch_step=2/2 micro_steps=9950 elapsed=1.3s lr=2.000000e-03 loss=0.1452 loss_recon=0.1452 loss_meanflow=0.0000 mean_model_t=0.2311 mean_corrupt_t=0.2311 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9600 corrupt_frac=0.6800 acc_corrupt=0.9559 loss_corrupt=0.1452 wrong_frac=0.7868 init_acc_corrupt=0.1544 acc_corrupt_t_0p0_0p2=0.9346 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.4103 out_g_norm=0.7348 acc_corrupt_t_0p2_0p4=0.9747 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=0.3819 init_gold_top10=0.5714 init_gold_top100=1.0000 +step=10000 epoch=5000/5000 epoch_step=2/2 micro_steps=10000 elapsed=1.3s lr=2.000000e-03 loss=0.0418 loss_recon=0.0418 loss_meanflow=0.0000 mean_model_t=0.2013 mean_corrupt_t=0.2013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=0.7000 acc_corrupt=0.9821 loss_corrupt=0.0418 wrong_frac=0.7750 init_acc_corrupt=0.1643 acc_corrupt_t_0p0_0p2=0.9708 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.4219 out_g_norm=0.6307 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=0.0000 init_gold_top10=0.3333 init_gold_top100=1.0000 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_linear_soft_kl_onehot_20260517_train8ctx8_overfit.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_linear_soft_kl_onehot_20260517_train8ctx8_overfit.log new file mode 100644 index 0000000000000000000000000000000000000000..5682ff632b6585ff5033748a0db908c8d7a9cc0e --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n8_linear_soft_kl_onehot_20260517_train8ctx8_overfit.log @@ -0,0 +1,326 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 50257, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_n8_linear_soft_kl_onehot_20260517_train8ctx8_overfit", + "batch_size": 1, + "grad_accum": 1, + "effective_batch_size": 4, + "global_batch_size": 4, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 2, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 169453056, + "muon_adam_param_count": 122368, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "blocks.3.attn_qkv.weight", + "blocks.3.attn_out.weight", + "blocks.3.mlp.0.weight", + "blocks.3.mlp.2.weight", + "blocks.3.adaLN_modulation.weight", + "blocks.4.attn_qkv.weight", + "blocks.4.attn_out.weight", + "blocks.4.mlp.0.weight", + "blocks.4.mlp.2.weight", + "blocks.4.adaLN_modulation.weight", + "blocks.5.attn_qkv.weight", + "blocks.5.attn_out.weight", + "blocks.5.mlp.0.weight", + "blocks.5.mlp.2.weight", + "blocks.5.adaLN_modulation.weight", + "blocks.6.attn_qkv.weight", + "blocks.6.attn_out.weight", + "blocks.6.mlp.0.weight", + "blocks.6.mlp.2.weight", + "blocks.6.adaLN_modulation.weight", + "blocks.7.attn_qkv.weight", + "blocks.7.attn_out.weight", + "blocks.7.mlp.0.weight", + "blocks.7.mlp.2.weight", + "blocks.7.adaLN_modulation.weight", + "blocks.8.attn_qkv.weight", + "blocks.8.attn_out.weight", + "blocks.8.mlp.0.weight", + "blocks.8.mlp.2.weight", + "blocks.8.adaLN_modulation.weight", + "blocks.9.attn_qkv.weight", + "blocks.9.attn_out.weight", + "blocks.9.mlp.0.weight", + "blocks.9.mlp.2.weight", + "blocks.9.adaLN_modulation.weight", + "blocks.10.attn_qkv.weight", + "blocks.10.attn_out.weight", + "blocks.10.mlp.0.weight", + "blocks.10.mlp.2.weight", + "blocks.10.adaLN_modulation.weight", + "blocks.11.attn_qkv.weight", + "blocks.11.attn_out.weight", + "blocks.11.mlp.0.weight", + "blocks.11.mlp.2.weight", + "blocks.11.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "blocks.3.norm1.weight", + "blocks.3.norm2.weight", + "blocks.3.mlp.0.bias", + "blocks.3.mlp.2.bias", + "blocks.3.adaLN_modulation.bias", + "blocks.4.norm1.weight", + "blocks.4.norm2.weight", + "blocks.4.mlp.0.bias", + "blocks.4.mlp.2.bias", + "blocks.4.adaLN_modulation.bias", + "blocks.5.norm1.weight", + "blocks.5.norm2.weight", + "blocks.5.mlp.0.bias", + "blocks.5.mlp.2.bias", + "blocks.5.adaLN_modulation.bias", + "blocks.6.norm1.weight", + "blocks.6.norm2.weight", + "blocks.6.mlp.0.bias", + "blocks.6.mlp.2.bias", + "blocks.6.adaLN_modulation.bias", + "blocks.7.norm1.weight", + "blocks.7.norm2.weight", + "blocks.7.mlp.0.bias", + "blocks.7.mlp.2.bias", + "blocks.7.adaLN_modulation.bias", + "blocks.8.norm1.weight", + "blocks.8.norm2.weight", + "blocks.8.mlp.0.bias", + "blocks.8.mlp.2.bias", + "blocks.8.adaLN_modulation.bias", + "blocks.9.norm1.weight", + "blocks.9.norm2.weight", + "blocks.9.mlp.0.bias", + "blocks.9.mlp.2.bias", + "blocks.9.adaLN_modulation.bias", + "blocks.10.norm1.weight", + "blocks.10.norm2.weight", + "blocks.10.mlp.0.bias", + "blocks.10.mlp.2.bias", + "blocks.10.adaLN_modulation.bias", + "blocks.11.norm1.weight", + "blocks.11.norm2.weight", + "blocks.11.mlp.0.bias", + "blocks.11.mlp.2.bias", + "blocks.11.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "linear_soft_kl", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 1.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len8_train8_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 10, + "resume_path": "" +} +step=10 epoch=5/250 epoch_step=2/2 micro_steps=10 elapsed=2.1s lr=2.000000e-03 loss=1.4378 loss_recon=1.4378 loss_meanflow=0.0000 mean_model_t=0.1662 mean_corrupt_t=0.1662 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1662 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2500 corrupt_frac=0.7125 acc_corrupt=0.1228 loss_corrupt=2.1079 wrong_frac=0.7895 init_acc_corrupt=0.0877 acc_corrupt_t_0p0_0p2=0.0513 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.0168 out_g_norm=1.4755 acc_corrupt_t_0p2_0p4=0.2778 corrupt_frac_t_0p2_0p4=1.0000 loss_all=10.7422 init_gold_top10=0.5000 init_gold_top100=0.5000 +step=20 epoch=10/250 epoch_step=2/2 micro_steps=20 elapsed=5.4s lr=2.000000e-03 loss=1.6261 loss_recon=1.6261 loss_meanflow=0.0000 mean_model_t=0.1939 mean_corrupt_t=0.1939 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1939 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3000 corrupt_frac=0.6375 acc_corrupt=0.1569 loss_corrupt=2.2662 wrong_frac=0.8824 init_acc_corrupt=0.0588 acc_corrupt_t_0p0_0p2=0.1538 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.1012 out_g_norm=1.7398 acc_corrupt_t_0p2_0p4=0.1579 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.1667 corrupt_frac_t_0p4_0p6=1.0000 loss_all=10.6484 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=30 epoch=15/250 epoch_step=2/2 micro_steps=30 elapsed=5.0s lr=2.000000e-03 loss=1.4998 loss_recon=1.4998 loss_meanflow=0.0000 mean_model_t=0.1869 mean_corrupt_t=0.1869 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1869 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1875 corrupt_frac=0.7125 acc_corrupt=0.1579 loss_corrupt=2.0982 wrong_frac=0.8772 init_acc_corrupt=0.0175 acc_corrupt_t_0p0_0p2=0.1379 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.2113 out_g_norm=1.9543 acc_corrupt_t_0p2_0p4=0.1786 corrupt_frac_t_0p2_0p4=1.0000 loss_all=10.3594 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=40 epoch=20/250 epoch_step=2/2 micro_steps=40 elapsed=4.8s lr=2.000000e-03 loss=2.1429 loss_recon=2.1429 loss_meanflow=0.0000 mean_model_t=0.2433 mean_corrupt_t=0.2433 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2433 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2750 corrupt_frac=0.6625 acc_corrupt=0.1887 loss_corrupt=2.8585 wrong_frac=0.7925 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.2222 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.3226 out_g_norm=1.9545 acc_corrupt_t_0p2_0p4=0.1724 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.1667 corrupt_frac_t_0p6_0p8=1.0000 loss_all=10.1250 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=50 epoch=25/250 epoch_step=2/2 micro_steps=50 elapsed=5.3s lr=2.000000e-03 loss=2.0644 loss_recon=2.0644 loss_meanflow=0.0000 mean_model_t=0.2392 mean_corrupt_t=0.2392 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2392 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2500 corrupt_frac=0.7875 acc_corrupt=0.1111 loss_corrupt=3.0162 wrong_frac=0.7778 init_acc_corrupt=0.1111 acc_corrupt_t_0p0_0p2=0.0833 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.4197 out_g_norm=1.8317 acc_corrupt_t_0p4_0p6=0.1429 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.1200 corrupt_frac_t_0p2_0p4=1.0000 loss_all=10.3516 init_gold_top10=0.2857 init_gold_top100=0.4286 +step=60 epoch=30/250 epoch_step=2/2 micro_steps=60 elapsed=3.9s lr=2.000000e-03 loss=0.6981 loss_recon=0.6981 loss_meanflow=0.0000 mean_model_t=0.0956 mean_corrupt_t=0.0956 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0956 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3125 corrupt_frac=0.7125 acc_corrupt=0.1404 loss_corrupt=1.7183 wrong_frac=0.8596 init_acc_corrupt=0.0175 acc_corrupt_t_0p0_0p2=0.1400 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.5080 out_g_norm=1.7802 acc_corrupt_t_0p2_0p4=0.1429 corrupt_frac_t_0p2_0p4=1.0000 loss_all=9.8359 init_gold_top10=0.0000 init_gold_top100=0.1667 +step=70 epoch=35/250 epoch_step=2/2 micro_steps=70 elapsed=5.3s lr=2.000000e-03 loss=1.2525 loss_recon=1.2525 loss_meanflow=0.0000 mean_model_t=0.1739 mean_corrupt_t=0.1739 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1739 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4500 corrupt_frac=0.6875 acc_corrupt=0.2909 loss_corrupt=2.1904 wrong_frac=0.8000 init_acc_corrupt=0.1091 acc_corrupt_t_0p0_0p2=0.2051 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.5936 out_g_norm=2.0018 acc_corrupt_t_0p2_0p4=0.5000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=9.6328 init_gold_top10=0.1667 init_gold_top100=0.1667 +step=80 epoch=40/250 epoch_step=2/2 micro_steps=80 elapsed=4.3s lr=2.000000e-03 loss=1.0250 loss_recon=1.0250 loss_meanflow=0.0000 mean_model_t=0.1471 mean_corrupt_t=0.1471 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1471 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3375 corrupt_frac=0.7250 acc_corrupt=0.2414 loss_corrupt=1.5685 wrong_frac=0.8103 init_acc_corrupt=0.0517 acc_corrupt_t_0p0_0p2=0.2308 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.6759 out_g_norm=1.6317 acc_corrupt_t_0p2_0p4=0.3333 corrupt_frac_t_0p2_0p4=1.0000 loss_all=8.8125 init_gold_top10=0.2500 init_gold_top100=0.2500 +step=90 epoch=45/250 epoch_step=2/2 micro_steps=90 elapsed=4.3s lr=2.000000e-03 loss=2.0293 loss_recon=2.0293 loss_meanflow=0.0000 mean_model_t=0.2610 mean_corrupt_t=0.2610 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2610 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4125 corrupt_frac=0.7125 acc_corrupt=0.2281 loss_corrupt=2.8313 wrong_frac=0.7719 init_acc_corrupt=0.2105 acc_corrupt_t_0p4_0p6=0.3333 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=0.7576 out_g_norm=2.3043 acc_corrupt_t_0p2_0p4=0.2083 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p0_0p2=0.1905 corrupt_frac_t_0p0_0p2=1.0000 loss_all=9.1992 init_gold_top10=0.1667 init_gold_top100=0.1667 +step=100 epoch=50/250 epoch_step=2/2 micro_steps=100 elapsed=5.3s lr=2.000000e-03 loss=1.3626 loss_recon=1.3626 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2061 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4375 corrupt_frac=0.7000 acc_corrupt=0.2321 loss_corrupt=1.9125 wrong_frac=0.8393 init_acc_corrupt=0.0893 acc_corrupt_t_0p0_0p2=0.2059 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.8440 out_g_norm=2.1251 acc_corrupt_t_0p2_0p4=0.2727 corrupt_frac_t_0p2_0p4=1.0000 loss_all=8.1016 init_gold_top10=0.2500 init_gold_top100=0.2500 +step=110 epoch=55/250 epoch_step=2/2 micro_steps=110 elapsed=4.3s lr=2.000000e-03 loss=1.2112 loss_recon=1.2112 loss_meanflow=0.0000 mean_model_t=0.1964 mean_corrupt_t=0.1964 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1964 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5375 corrupt_frac=0.6625 acc_corrupt=0.3208 loss_corrupt=2.1580 wrong_frac=0.6981 init_acc_corrupt=0.1509 acc_corrupt_t_0p2_0p4=0.3182 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=0.9333 out_g_norm=2.1929 acc_corrupt_t_0p0_0p2=0.2000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.8333 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.6875 init_gold_top10=0.2857 init_gold_top100=0.2857 +step=120 epoch=60/250 epoch_step=2/2 micro_steps=120 elapsed=3.9s lr=2.000000e-03 loss=1.3013 loss_recon=1.3013 loss_meanflow=0.0000 mean_model_t=0.2019 mean_corrupt_t=0.2019 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2019 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4750 corrupt_frac=0.7500 acc_corrupt=0.3000 loss_corrupt=1.7847 wrong_frac=0.7500 init_acc_corrupt=0.2167 acc_corrupt_t_0p0_0p2=0.1786 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.0233 out_g_norm=1.7968 acc_corrupt_t_0p2_0p4=0.3077 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.8333 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.5352 init_gold_top10=0.6667 init_gold_top100=0.6667 +step=130 epoch=65/250 epoch_step=2/2 micro_steps=130 elapsed=5.0s lr=2.000000e-03 loss=0.9233 loss_recon=0.9233 loss_meanflow=0.0000 mean_model_t=0.1694 mean_corrupt_t=0.1694 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1694 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4125 corrupt_frac=0.7125 acc_corrupt=0.2105 loss_corrupt=1.8001 wrong_frac=0.7895 init_acc_corrupt=0.0175 acc_corrupt_t_0p0_0p2=0.1944 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.1115 out_g_norm=2.2608 acc_corrupt_t_0p2_0p4=0.2381 corrupt_frac_t_0p2_0p4=1.0000 loss_all=7.4980 init_gold_top10=0.3333 init_gold_top100=0.3333 +step=140 epoch=70/250 epoch_step=2/2 micro_steps=140 elapsed=4.3s lr=2.000000e-03 loss=1.2486 loss_recon=1.2486 loss_meanflow=0.0000 mean_model_t=0.2216 mean_corrupt_t=0.2216 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2216 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4000 corrupt_frac=0.8000 acc_corrupt=0.2500 loss_corrupt=1.7026 wrong_frac=0.7969 init_acc_corrupt=0.1094 acc_corrupt_t_0p2_0p4=0.1579 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.2048 out_g_norm=2.0386 acc_corrupt_t_0p4_0p6=0.7143 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p0_0p2=0.2105 corrupt_frac_t_0p0_0p2=1.0000 loss_all=5.8379 init_gold_top10=0.3333 init_gold_top100=0.3333 +step=150 epoch=75/250 epoch_step=2/2 micro_steps=150 elapsed=3.9s lr=2.000000e-03 loss=0.9679 loss_recon=0.9679 loss_meanflow=0.0000 mean_model_t=0.2049 mean_corrupt_t=0.2049 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2049 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5250 corrupt_frac=0.6875 acc_corrupt=0.3091 loss_corrupt=1.9291 wrong_frac=0.8727 init_acc_corrupt=0.0727 acc_corrupt_t_0p4_0p6=0.3333 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=1.3014 out_g_norm=2.1012 acc_corrupt_t_0p2_0p4=0.4286 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p0_0p2=0.2143 corrupt_frac_t_0p0_0p2=1.0000 loss_all=7.2578 init_gold_top10=0.0000 init_gold_top100=0.2857 +step=160 epoch=80/250 epoch_step=2/2 micro_steps=160 elapsed=4.4s lr=2.000000e-03 loss=0.9315 loss_recon=0.9315 loss_meanflow=0.0000 mean_model_t=0.2270 mean_corrupt_t=0.2270 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2270 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5625 corrupt_frac=0.7125 acc_corrupt=0.3860 loss_corrupt=1.5416 wrong_frac=0.7018 init_acc_corrupt=0.1754 acc_corrupt_t_0p0_0p2=0.3750 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.3993 out_g_norm=2.1721 acc_corrupt_t_0p2_0p4=0.2632 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.8333 corrupt_frac_t_0p6_0p8=1.0000 loss_all=5.0039 init_gold_top10=0.1667 init_gold_top100=0.1667 +step=170 epoch=85/250 epoch_step=2/2 micro_steps=170 elapsed=4.3s lr=2.000000e-03 loss=0.9990 loss_recon=0.9990 loss_meanflow=0.0000 mean_model_t=0.2447 mean_corrupt_t=0.2447 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2447 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5875 corrupt_frac=0.6500 acc_corrupt=0.3654 loss_corrupt=1.2837 wrong_frac=0.8654 init_acc_corrupt=0.0962 acc_corrupt_t_0p2_0p4=0.3793 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.4993 out_g_norm=2.1622 acc_corrupt_t_0p4_0p6=0.5714 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p0_0p2=0.2500 corrupt_frac_t_0p0_0p2=1.0000 loss_all=2.9458 init_gold_top10=0.0000 init_gold_top100=0.2500 +step=180 epoch=90/250 epoch_step=2/2 micro_steps=180 elapsed=3.9s lr=2.000000e-03 loss=0.7330 loss_recon=0.7330 loss_meanflow=0.0000 mean_model_t=0.2271 mean_corrupt_t=0.2271 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2271 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6000 corrupt_frac=0.6375 acc_corrupt=0.3725 loss_corrupt=1.1424 wrong_frac=0.7843 init_acc_corrupt=0.1569 acc_corrupt_t_0p2_0p4=0.4444 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.5973 out_g_norm=2.0392 acc_corrupt_t_0p0_0p2=0.2963 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.5000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.4272 init_gold_top10=0.2500 init_gold_top100=0.2500 +step=190 epoch=95/250 epoch_step=2/2 micro_steps=190 elapsed=4.4s lr=2.000000e-03 loss=0.6392 loss_recon=0.6392 loss_meanflow=0.0000 mean_model_t=0.1793 mean_corrupt_t=0.1793 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1793 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4500 corrupt_frac=0.7125 acc_corrupt=0.2456 loss_corrupt=1.4030 wrong_frac=0.8070 init_acc_corrupt=0.0702 acc_corrupt_t_0p0_0p2=0.1429 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.6917 out_g_norm=2.2318 acc_corrupt_t_0p2_0p4=0.3636 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.1974 init_gold_top10=0.5000 init_gold_top100=0.5000 +step=200 epoch=100/250 epoch_step=2/2 micro_steps=200 elapsed=4.3s lr=2.000000e-03 loss=0.6127 loss_recon=0.6127 loss_meanflow=0.0000 mean_model_t=0.1869 mean_corrupt_t=0.1869 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1869 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5125 corrupt_frac=0.7625 acc_corrupt=0.3607 loss_corrupt=0.9250 wrong_frac=0.8033 init_acc_corrupt=0.1148 acc_corrupt_t_0p2_0p4=0.5000 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.7771 out_g_norm=2.0441 acc_corrupt_t_0p0_0p2=0.2069 corrupt_frac_t_0p0_0p2=1.0000 loss_all=2.7065 init_gold_top10=0.0000 init_gold_top100=0.2500 +step=210 epoch=105/250 epoch_step=2/2 micro_steps=210 elapsed=3.9s lr=2.000000e-03 loss=0.3842 loss_recon=0.3842 loss_meanflow=0.0000 mean_model_t=0.1656 mean_corrupt_t=0.1656 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1656 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4875 corrupt_frac=0.7625 acc_corrupt=0.3279 loss_corrupt=0.8346 wrong_frac=0.8689 init_acc_corrupt=0.0656 acc_corrupt_t_0p0_0p2=0.2391 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.8511 out_g_norm=1.9414 acc_corrupt_t_0p2_0p4=0.5455 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.7500 corrupt_frac_t_0p4_0p6=1.0000 loss_all=3.7437 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=220 epoch=110/250 epoch_step=2/2 micro_steps=220 elapsed=4.4s lr=2.000000e-03 loss=0.5714 loss_recon=0.5714 loss_meanflow=0.0000 mean_model_t=0.2338 mean_corrupt_t=0.2338 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2338 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5750 corrupt_frac=0.7125 acc_corrupt=0.4035 loss_corrupt=0.9256 wrong_frac=0.7193 init_acc_corrupt=0.1754 acc_corrupt_t_0p2_0p4=0.3871 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.9223 out_g_norm=2.8620 acc_corrupt_t_0p0_0p2=0.3500 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.6667 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.8745 init_gold_top10=0.4286 init_gold_top100=0.4286 +step=230 epoch=115/250 epoch_step=2/2 micro_steps=230 elapsed=4.3s lr=2.000000e-03 loss=0.3316 loss_recon=0.3316 loss_meanflow=0.0000 mean_model_t=0.1464 mean_corrupt_t=0.1464 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1464 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4875 corrupt_frac=0.6875 acc_corrupt=0.2545 loss_corrupt=0.6993 wrong_frac=0.8545 init_acc_corrupt=0.0545 acc_corrupt_t_0p0_0p2=0.2500 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.9838 out_g_norm=1.9833 acc_corrupt_t_0p2_0p4=0.2632 corrupt_frac_t_0p2_0p4=1.0000 loss_all=3.0620 init_gold_top10=0.3333 init_gold_top100=0.3333 +step=240 epoch=120/250 epoch_step=2/2 micro_steps=240 elapsed=3.9s lr=2.000000e-03 loss=0.5485 loss_recon=0.5485 loss_meanflow=0.0000 mean_model_t=0.1917 mean_corrupt_t=0.1917 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1917 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4250 corrupt_frac=0.8125 acc_corrupt=0.2923 loss_corrupt=1.0804 wrong_frac=0.8769 init_acc_corrupt=0.0462 acc_corrupt_t_0p2_0p4=0.1579 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0305 out_g_norm=2.2724 acc_corrupt_t_0p4_0p6=0.8333 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p0_0p2=0.2750 corrupt_frac_t_0p0_0p2=1.0000 loss_all=4.9873 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=250 epoch=125/250 epoch_step=2/2 micro_steps=250 elapsed=4.3s lr=2.000000e-03 loss=0.5894 loss_recon=0.5894 loss_meanflow=0.0000 mean_model_t=0.2324 mean_corrupt_t=0.2324 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2324 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5000 corrupt_frac=0.7375 acc_corrupt=0.3220 loss_corrupt=1.1833 wrong_frac=0.7966 init_acc_corrupt=0.1356 acc_corrupt_t_0p2_0p4=0.3571 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0676 out_g_norm=2.6213 acc_corrupt_t_0p0_0p2=0.2353 corrupt_frac_t_0p0_0p2=1.0000 loss_all=5.5378 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=260 epoch=130/250 epoch_step=2/2 micro_steps=260 elapsed=4.3s lr=2.000000e-03 loss=0.4236 loss_recon=0.4236 loss_meanflow=0.0000 mean_model_t=0.2037 mean_corrupt_t=0.2037 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2037 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6125 corrupt_frac=0.7000 acc_corrupt=0.4464 loss_corrupt=0.6426 wrong_frac=0.7857 init_acc_corrupt=0.0893 acc_corrupt_t_0p0_0p2=0.2500 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0902 out_g_norm=2.7820 acc_corrupt_t_0p2_0p4=0.6429 corrupt_frac_t_0p2_0p4=1.0000 loss_all=1.9424 init_gold_top10=0.0000 init_gold_top100=0.2500 +step=270 epoch=135/250 epoch_step=2/2 micro_steps=270 elapsed=3.9s lr=2.000000e-03 loss=0.4216 loss_recon=0.4216 loss_meanflow=0.0000 mean_model_t=0.2494 mean_corrupt_t=0.2494 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2494 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6375 corrupt_frac=0.6375 acc_corrupt=0.4314 loss_corrupt=0.6980 wrong_frac=0.8235 init_acc_corrupt=0.1373 acc_corrupt_t_0p0_0p2=0.3103 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0985 out_g_norm=2.1731 acc_corrupt_t_0p2_0p4=0.6667 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.8916 init_gold_top10=0.3333 init_gold_top100=0.3333 +step=280 epoch=140/250 epoch_step=2/2 micro_steps=280 elapsed=4.4s lr=2.000000e-03 loss=0.4478 loss_recon=0.4478 loss_meanflow=0.0000 mean_model_t=0.2913 mean_corrupt_t=0.2913 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2913 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6750 corrupt_frac=0.6125 acc_corrupt=0.4694 loss_corrupt=0.6763 wrong_frac=0.7143 init_acc_corrupt=0.2041 acc_corrupt_t_0p0_0p2=0.3600 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.1126 out_g_norm=2.5709 acc_corrupt_t_0p6_0p8=0.7500 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p2_0p4=0.5833 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.2793 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=290 epoch=145/250 epoch_step=2/2 micro_steps=290 elapsed=4.3s lr=2.000000e-03 loss=0.2915 loss_recon=0.2915 loss_meanflow=0.0000 mean_model_t=0.1777 mean_corrupt_t=0.1777 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1777 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6000 corrupt_frac=0.7125 acc_corrupt=0.4386 loss_corrupt=0.5437 wrong_frac=0.8947 init_acc_corrupt=0.0526 acc_corrupt_t_0p0_0p2=0.3333 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.1215 out_g_norm=2.7226 acc_corrupt_t_0p2_0p4=0.5833 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.3824 init_gold_top10=0.1667 init_gold_top100=0.3333 +step=300 epoch=150/250 epoch_step=2/2 micro_steps=300 elapsed=3.9s lr=2.000000e-03 loss=0.4178 loss_recon=0.4178 loss_meanflow=0.0000 mean_model_t=0.2061 mean_corrupt_t=0.2061 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2061 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6125 corrupt_frac=0.7375 acc_corrupt=0.4746 loss_corrupt=0.7239 wrong_frac=0.7627 init_acc_corrupt=0.0678 acc_corrupt_t_0p0_0p2=0.6154 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.1224 out_g_norm=2.1706 acc_corrupt_t_0p2_0p4=0.3636 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.5565 init_gold_top10=0.1429 init_gold_top100=0.5714 +step=310 epoch=155/250 epoch_step=2/2 micro_steps=310 elapsed=4.9s lr=2.000000e-03 loss=0.4163 loss_recon=0.4163 loss_meanflow=0.0000 mean_model_t=0.1841 mean_corrupt_t=0.1841 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1841 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6125 corrupt_frac=0.7000 acc_corrupt=0.4464 loss_corrupt=0.6763 wrong_frac=0.7857 init_acc_corrupt=0.1429 acc_corrupt_t_0p0_0p2=0.3889 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.1090 out_g_norm=2.5876 acc_corrupt_t_0p4_0p6=0.5714 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.5385 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.2671 init_gold_top10=0.5714 init_gold_top100=0.5714 +step=320 epoch=160/250 epoch_step=2/2 micro_steps=320 elapsed=4.2s lr=2.000000e-03 loss=0.4162 loss_recon=0.4162 loss_meanflow=0.0000 mean_model_t=0.2237 mean_corrupt_t=0.2237 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2237 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6500 corrupt_frac=0.7625 acc_corrupt=0.5410 loss_corrupt=0.6050 wrong_frac=0.7213 init_acc_corrupt=0.1803 acc_corrupt_t_0p0_0p2=0.5185 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0998 out_g_norm=2.0437 acc_corrupt_t_0p2_0p4=0.5556 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5714 corrupt_frac_t_0p4_0p6=1.0000 loss_all=2.4873 init_gold_top10=0.2500 init_gold_top100=0.5000 +step=330 epoch=165/250 epoch_step=2/2 micro_steps=330 elapsed=4.0s lr=2.000000e-03 loss=0.3596 loss_recon=0.3596 loss_meanflow=0.0000 mean_model_t=0.1733 mean_corrupt_t=0.1733 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1733 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4375 corrupt_frac=0.7750 acc_corrupt=0.2742 loss_corrupt=0.9846 wrong_frac=0.8710 init_acc_corrupt=0.0968 acc_corrupt_t_0p0_0p2=0.1277 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0984 out_g_norm=2.1073 acc_corrupt_t_0p2_0p4=0.7333 corrupt_frac_t_0p2_0p4=1.0000 loss_all=5.2998 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=340 epoch=170/250 epoch_step=2/2 micro_steps=340 elapsed=4.4s lr=2.000000e-03 loss=0.2857 loss_recon=0.2857 loss_meanflow=0.0000 mean_model_t=0.1706 mean_corrupt_t=0.1706 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1706 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6625 corrupt_frac=0.6875 acc_corrupt=0.5273 loss_corrupt=0.4981 wrong_frac=0.8364 init_acc_corrupt=0.0545 acc_corrupt_t_0p4_0p6=0.7500 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=2.0925 out_g_norm=2.0339 acc_corrupt_t_0p0_0p2=0.5000 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p2_0p4=0.5714 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.0654 init_gold_top10=0.0000 init_gold_top100=0.0000 +step=350 epoch=175/250 epoch_step=2/2 micro_steps=350 elapsed=4.2s lr=2.000000e-03 loss=0.1813 loss_recon=0.1813 loss_meanflow=0.0000 mean_model_t=0.1718 mean_corrupt_t=0.1718 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1718 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7750 corrupt_frac=0.7000 acc_corrupt=0.6786 loss_corrupt=0.4732 wrong_frac=0.8214 init_acc_corrupt=0.1607 acc_corrupt_t_0p0_0p2=0.5500 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0854 out_g_norm=2.3255 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.5088 init_gold_top10=0.1667 init_gold_top100=0.3333 +step=360 epoch=180/250 epoch_step=2/2 micro_steps=360 elapsed=3.9s lr=2.000000e-03 loss=0.1510 loss_recon=0.1510 loss_meanflow=0.0000 mean_model_t=0.2313 mean_corrupt_t=0.2313 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2313 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7875 corrupt_frac=0.7125 acc_corrupt=0.7018 loss_corrupt=0.3167 wrong_frac=0.7368 init_acc_corrupt=0.1930 acc_corrupt_t_0p2_0p4=0.8571 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0792 out_g_norm=1.7478 acc_corrupt_t_0p0_0p2=0.4828 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.8887 init_gold_top10=0.0000 init_gold_top100=0.5000 +step=370 epoch=185/250 epoch_step=2/2 micro_steps=370 elapsed=4.3s lr=2.000000e-03 loss=0.2450 loss_recon=0.2450 loss_meanflow=0.0000 mean_model_t=0.1875 mean_corrupt_t=0.1875 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1875 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8000 corrupt_frac=0.6250 acc_corrupt=0.6800 loss_corrupt=0.3616 wrong_frac=0.8000 init_acc_corrupt=0.1000 acc_corrupt_t_0p0_0p2=0.7586 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0847 out_g_norm=2.1802 acc_corrupt_t_0p2_0p4=0.5714 corrupt_frac_t_0p2_0p4=1.0000 loss_all=1.4507 init_gold_top10=0.2500 init_gold_top100=0.2500 +step=380 epoch=190/250 epoch_step=2/2 micro_steps=380 elapsed=4.3s lr=2.000000e-03 loss=0.1449 loss_recon=0.1449 loss_meanflow=0.0000 mean_model_t=0.1769 mean_corrupt_t=0.1769 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1769 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8000 corrupt_frac=0.6500 acc_corrupt=0.6923 loss_corrupt=0.3072 wrong_frac=0.7692 init_acc_corrupt=0.0962 acc_corrupt_t_0p0_0p2=0.6250 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0835 out_g_norm=1.5933 acc_corrupt_t_0p2_0p4=0.9167 corrupt_frac_t_0p2_0p4=1.0000 loss_all=1.4590 init_gold_top10=0.5000 init_gold_top100=0.5000 +step=390 epoch=195/250 epoch_step=2/2 micro_steps=390 elapsed=3.9s lr=2.000000e-03 loss=0.1706 loss_recon=0.1706 loss_meanflow=0.0000 mean_model_t=0.2492 mean_corrupt_t=0.2492 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2492 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8250 corrupt_frac=0.7750 acc_corrupt=0.7742 loss_corrupt=0.3738 wrong_frac=0.7097 init_acc_corrupt=0.0968 acc_corrupt_t_0p2_0p4=0.7843 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0944 out_g_norm=2.0398 acc_corrupt_t_0p0_0p2=0.7273 corrupt_frac_t_0p0_0p2=1.0000 loss_all=1.9468 init_gold_top10=0.1429 init_gold_top100=0.1429 +step=400 epoch=200/250 epoch_step=2/2 micro_steps=400 elapsed=4.3s lr=2.000000e-03 loss=0.1752 loss_recon=0.1752 loss_meanflow=0.0000 mean_model_t=0.2467 mean_corrupt_t=0.2467 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2467 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8000 corrupt_frac=0.7750 acc_corrupt=0.7419 loss_corrupt=0.3860 wrong_frac=0.7903 init_acc_corrupt=0.1613 acc_corrupt_t_0p0_0p2=0.5000 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0922 out_g_norm=1.7354 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.8750 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.0479 init_gold_top10=0.4286 init_gold_top100=0.4286 +step=410 epoch=205/250 epoch_step=2/2 micro_steps=410 elapsed=4.3s lr=2.000000e-03 loss=0.2606 loss_recon=0.2606 loss_meanflow=0.0000 mean_model_t=0.2461 mean_corrupt_t=0.2461 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2461 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7625 corrupt_frac=0.6250 acc_corrupt=0.6200 loss_corrupt=1.0780 wrong_frac=0.7600 init_acc_corrupt=0.1200 acc_corrupt_t_0p0_0p2=0.4286 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0802 out_g_norm=2.0628 acc_corrupt_t_0p2_0p4=0.8000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.9167 corrupt_frac_t_0p4_0p6=1.0000 loss_all=5.3184 init_gold_top10=0.0000 init_gold_top100=0.2857 +step=420 epoch=210/250 epoch_step=2/2 micro_steps=420 elapsed=3.9s lr=2.000000e-03 loss=0.2090 loss_recon=0.2090 loss_meanflow=0.0000 mean_model_t=0.2875 mean_corrupt_t=0.2875 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2875 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8750 corrupt_frac=0.7625 acc_corrupt=0.8361 loss_corrupt=0.3340 wrong_frac=0.7049 init_acc_corrupt=0.2459 acc_corrupt_t_0p0_0p2=0.7407 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0755 out_g_norm=1.5891 acc_corrupt_t_0p4_0p6=0.9000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.9286 corrupt_frac_t_0p2_0p4=1.0000 loss_all=1.5884 init_gold_top10=0.5000 init_gold_top100=0.5000 +step=430 epoch=215/250 epoch_step=2/2 micro_steps=430 elapsed=4.3s lr=2.000000e-03 loss=0.2835 loss_recon=0.2835 loss_meanflow=0.0000 mean_model_t=0.2302 mean_corrupt_t=0.2302 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2302 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8000 corrupt_frac=0.6875 acc_corrupt=0.7091 loss_corrupt=0.9789 wrong_frac=0.8727 init_acc_corrupt=0.0727 acc_corrupt_t_0p0_0p2=0.7778 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0719 out_g_norm=1.7877 acc_corrupt_t_0p2_0p4=0.5833 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=5.6514 init_gold_top10=0.0000 init_gold_top100=0.2857 +step=440 epoch=220/250 epoch_step=2/2 micro_steps=440 elapsed=4.3s lr=2.000000e-03 loss=0.1238 loss_recon=0.1238 loss_meanflow=0.0000 mean_model_t=0.2328 mean_corrupt_t=0.2328 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2328 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8625 corrupt_frac=0.6500 acc_corrupt=0.7885 loss_corrupt=0.6087 wrong_frac=0.7692 init_acc_corrupt=0.1731 acc_corrupt_t_0p0_0p2=0.6765 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0703 out_g_norm=1.9463 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=1.0000 loss_all=3.3848 init_gold_top10=0.0000 init_gold_top100=0.2857 +step=450 epoch=225/250 epoch_step=2/2 micro_steps=450 elapsed=3.9s lr=2.000000e-03 loss=0.1397 loss_recon=0.1397 loss_meanflow=0.0000 mean_model_t=0.1586 mean_corrupt_t=0.1586 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1586 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8875 corrupt_frac=0.6750 acc_corrupt=0.8333 loss_corrupt=0.4628 wrong_frac=0.8333 init_acc_corrupt=0.0185 acc_corrupt_t_0p2_0p4=0.6471 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0704 out_g_norm=1.4328 acc_corrupt_t_0p0_0p2=0.9189 corrupt_frac_t_0p0_0p2=1.0000 loss_all=2.4082 init_gold_top10=0.1429 init_gold_top100=0.1429 +step=460 epoch=230/250 epoch_step=2/2 micro_steps=460 elapsed=4.4s lr=2.000000e-03 loss=0.1377 loss_recon=0.1377 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2081 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9125 corrupt_frac=0.6375 acc_corrupt=0.8627 loss_corrupt=0.2271 wrong_frac=0.7255 init_acc_corrupt=0.1765 acc_corrupt_t_0p0_0p2=0.8437 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0668 out_g_norm=1.6259 acc_corrupt_t_0p2_0p4=0.9091 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.8750 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.0952 init_gold_top10=0.5000 init_gold_top100=0.5000 +step=470 epoch=235/250 epoch_step=2/2 micro_steps=470 elapsed=4.3s lr=2.000000e-03 loss=0.1759 loss_recon=0.1759 loss_meanflow=0.0000 mean_model_t=0.1753 mean_corrupt_t=0.1753 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.1753 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8500 corrupt_frac=0.7000 acc_corrupt=0.7857 loss_corrupt=0.4325 wrong_frac=0.8393 init_acc_corrupt=0.0893 acc_corrupt_t_0p0_0p2=0.7436 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0613 out_g_norm=1.8623 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p2_0p4=0.8182 corrupt_frac_t_0p2_0p4=1.0000 loss_all=1.9712 init_gold_top10=0.1429 init_gold_top100=0.1429 +step=480 epoch=240/250 epoch_step=2/2 micro_steps=480 elapsed=3.9s lr=2.000000e-03 loss=0.1247 loss_recon=0.1247 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2083 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=0.6500 acc_corrupt=0.9038 loss_corrupt=0.2654 wrong_frac=0.7500 init_acc_corrupt=0.1731 acc_corrupt_t_0p0_0p2=0.8750 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0751 out_g_norm=1.8762 acc_corrupt_t_0p2_0p4=0.9091 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.2505 init_gold_top10=0.0000 init_gold_top100=0.5000 +step=490 epoch=245/250 epoch_step=2/2 micro_steps=490 elapsed=4.8s lr=2.000000e-03 loss=0.0926 loss_recon=0.0926 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2059 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9625 corrupt_frac=0.7125 acc_corrupt=0.9474 loss_corrupt=0.3796 wrong_frac=0.7895 init_acc_corrupt=0.0877 acc_corrupt_t_0p0_0p2=0.9302 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.0723 out_g_norm=1.4834 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.5068 init_gold_top10=0.1667 init_gold_top100=0.1667 +step=500 epoch=250/250 epoch_step=2/2 micro_steps=500 elapsed=4.2s lr=2.000000e-03 loss=0.1136 loss_recon=0.1136 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.2092 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8250 corrupt_frac=0.8000 acc_corrupt=0.7813 loss_corrupt=0.2750 wrong_frac=0.7656 init_acc_corrupt=0.1563 acc_corrupt_t_0p2_0p4=0.8800 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=2.0694 out_g_norm=1.7062 acc_corrupt_t_0p0_0p2=0.6667 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=1.0000 loss_all=1.8398 init_gold_top10=0.3333 init_gold_top100=0.3333 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805.log new file mode 100644 index 0000000000000000000000000000000000000000..c466dbf052ee452029b8816830d65b0b61b5f503 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805.log @@ -0,0 +1,987 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.4s lr=2.000000e-03 loss=6.6495 loss_recon=6.6495 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1702 corrupt_frac=1.0000 acc_corrupt=0.1702 loss_corrupt=6.6495 wrong_frac=0.7917 init_acc_corrupt=0.2067 acc_corrupt_t_0p0_0p2=0.0878 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=0.2294 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.4491 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.6387 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=1.2350 out_g_norm=0.9734 acc_corrupt_t_0p8_1p0=0.8477 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.3486 init_gold_top10=0.2053 init_gold_top100=0.2800 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=3.7s lr=2.000000e-03 loss=5.8699 loss_recon=5.8699 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1640 corrupt_frac=1.0000 acc_corrupt=0.1640 loss_corrupt=5.8699 wrong_frac=0.7925 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.0976 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.2106 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.3899 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.6051 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=4.0392 out_g_norm=1.3294 acc_corrupt_t_0p8_1p0=0.7979 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.5159 init_gold_top10=0.2043 init_gold_top100=0.2783 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=3.7s lr=2.000000e-03 loss=5.1215 loss_recon=5.1215 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1844 corrupt_frac=1.0000 acc_corrupt=0.1844 loss_corrupt=5.1215 wrong_frac=0.7884 init_acc_corrupt=0.2102 acc_corrupt_t_0p0_0p2=0.1071 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=0.2350 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=0.4284 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.6246 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=6.5345 out_g_norm=0.7810 acc_corrupt_t_0p8_1p0=0.7812 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7567 init_gold_top10=0.2224 init_gold_top100=0.2981 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=3.7s lr=2.000000e-03 loss=4.3914 loss_recon=4.3914 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2275 corrupt_frac=1.0000 acc_corrupt=0.2275 loss_corrupt=4.3914 wrong_frac=0.7927 init_acc_corrupt=0.2056 acc_corrupt_t_0p0_0p2=0.1322 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=0.3118 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=0.4954 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=0.6688 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=8.2403 out_g_norm=0.4520 loss_all=3.9405 init_gold_top10=0.2197 init_gold_top100=0.2939 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=3.7s lr=2.000000e-03 loss=3.4904 loss_recon=3.4904 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2580 corrupt_frac=1.0000 acc_corrupt=0.2580 loss_corrupt=3.4904 wrong_frac=0.7904 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.1592 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=0.3420 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.5218 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=9.6795 out_g_norm=0.4054 acc_corrupt_t_0p6_0p8=0.6903 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.8594 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.9689 init_gold_top10=0.2225 init_gold_top100=0.2952 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.7s lr=2.000000e-03 loss=2.5069 loss_recon=2.5069 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3478 corrupt_frac=1.0000 acc_corrupt=0.3478 loss_corrupt=2.5069 wrong_frac=0.7910 init_acc_corrupt=0.2074 acc_corrupt_t_0p0_0p2=0.2274 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=0.4660 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.6365 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.7711 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=10.4910 out_g_norm=0.4782 acc_corrupt_t_0p8_1p0=0.8952 corrupt_frac_t_0p8_1p0=0.0094 loss_all=2.0582 init_gold_top10=0.2072 init_gold_top100=0.2815 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.7s lr=2.000000e-03 loss=1.6159 loss_recon=1.6159 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5261 corrupt_frac=1.0000 acc_corrupt=0.5261 loss_corrupt=1.6159 wrong_frac=0.7907 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.3700 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=0.6933 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=0.8285 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=11.0190 out_g_norm=0.5898 acc_corrupt_t_0p6_0p8=0.8964 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=0.9167 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3305 init_gold_top10=0.2117 init_gold_top100=0.2832 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.7s lr=2.000000e-03 loss=0.9537 loss_recon=0.9537 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7156 corrupt_frac=1.0000 acc_corrupt=0.7156 loss_corrupt=0.9537 wrong_frac=0.7897 init_acc_corrupt=0.2086 acc_corrupt_t_0p0_0p2=0.5634 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=0.8899 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=0.9571 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=0.9757 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=11.3997 out_g_norm=0.6929 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5768 init_gold_top10=0.2208 init_gold_top100=0.2931 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.7s lr=2.000000e-03 loss=0.5253 loss_recon=0.5253 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8455 corrupt_frac=1.0000 acc_corrupt=0.8455 loss_corrupt=0.5253 wrong_frac=0.7906 init_acc_corrupt=0.2078 acc_corrupt_t_0p0_0p2=0.7406 corrupt_frac_t_0p0_0p2=0.5584 acc_corrupt_t_0p2_0p4=0.9744 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9931 corrupt_frac_t_0p4_0p6=0.0801 acc_corrupt_t_0p6_0p8=0.9967 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=11.7285 out_g_norm=0.5915 acc_corrupt_t_0p8_1p0=0.9974 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3668 init_gold_top10=0.2190 init_gold_top100=0.2921 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.7s lr=2.000000e-03 loss=0.3128 loss_recon=0.3128 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9050 corrupt_frac=1.0000 acc_corrupt=0.9050 loss_corrupt=0.3128 wrong_frac=0.7908 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.8329 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.9944 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9985 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9982 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.9476 out_g_norm=0.4867 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2410 init_gold_top10=0.2187 init_gold_top100=0.2915 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=4.4s lr=2.000000e-03 loss=0.2449 loss_recon=0.2449 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9244 corrupt_frac=1.0000 acc_corrupt=0.9244 loss_corrupt=0.2449 wrong_frac=0.7917 init_acc_corrupt=0.2067 acc_corrupt_t_0p0_0p2=0.8659 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=0.9983 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.9992 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9987 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.0079 out_g_norm=0.3943 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2226 init_gold_top10=0.2053 init_gold_top100=0.2800 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=3.9s lr=2.000000e-03 loss=0.2175 loss_recon=0.2175 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9306 corrupt_frac=1.0000 acc_corrupt=0.9306 loss_corrupt=0.2175 wrong_frac=0.7925 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.8761 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9987 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=12.0269 out_g_norm=0.3266 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2401 init_gold_top10=0.2043 init_gold_top100=0.2783 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=3.9s lr=2.000000e-03 loss=0.1832 loss_recon=0.1832 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9416 corrupt_frac=1.0000 acc_corrupt=0.9416 loss_corrupt=0.1832 wrong_frac=0.7884 init_acc_corrupt=0.2102 acc_corrupt_t_0p0_0p2=0.8938 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.0197 out_g_norm=0.3052 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1627 init_gold_top10=0.2224 init_gold_top100=0.2981 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=3.9s lr=2.000000e-03 loss=0.1813 loss_recon=0.1813 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9413 corrupt_frac=1.0000 acc_corrupt=0.9413 loss_corrupt=0.1813 wrong_frac=0.7927 init_acc_corrupt=0.2056 acc_corrupt_t_0p0_0p2=0.8956 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=12.0342 out_g_norm=0.2857 loss_all=0.0911 init_gold_top10=0.2197 init_gold_top100=0.2939 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=3.9s lr=2.000000e-03 loss=0.1679 loss_recon=0.1679 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9460 corrupt_frac=1.0000 acc_corrupt=0.9460 loss_corrupt=0.1679 wrong_frac=0.7904 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.9026 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=12.0459 out_g_norm=0.2685 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1933 init_gold_top10=0.2225 init_gold_top100=0.2952 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=3.9s lr=2.000000e-03 loss=0.1488 loss_recon=0.1488 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9519 corrupt_frac=1.0000 acc_corrupt=0.9519 loss_corrupt=0.1488 wrong_frac=0.7910 init_acc_corrupt=0.2074 acc_corrupt_t_0p0_0p2=0.9142 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.0548 out_g_norm=0.2284 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0094 loss_all=0.1261 init_gold_top10=0.2072 init_gold_top100=0.2815 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=3.9s lr=2.000000e-03 loss=0.1446 loss_recon=0.1446 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9526 corrupt_frac=1.0000 acc_corrupt=0.9526 loss_corrupt=0.1446 wrong_frac=0.7907 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9147 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=12.0717 out_g_norm=0.1855 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1127 init_gold_top10=0.2117 init_gold_top100=0.2832 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=3.9s lr=2.000000e-03 loss=0.1385 loss_recon=0.1385 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9550 corrupt_frac=1.0000 acc_corrupt=0.9550 loss_corrupt=0.1385 wrong_frac=0.7897 init_acc_corrupt=0.2086 acc_corrupt_t_0p0_0p2=0.9186 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=12.0831 out_g_norm=0.1934 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0903 init_gold_top10=0.2208 init_gold_top100=0.2931 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=3.9s lr=2.000000e-03 loss=0.1373 loss_recon=0.1373 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9553 corrupt_frac=1.0000 acc_corrupt=0.9553 loss_corrupt=0.1373 wrong_frac=0.7906 init_acc_corrupt=0.2078 acc_corrupt_t_0p0_0p2=0.9199 corrupt_frac_t_0p0_0p2=0.5584 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0801 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.0937 out_g_norm=0.1728 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1211 init_gold_top10=0.2190 init_gold_top100=0.2921 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=3.9s lr=2.000000e-03 loss=0.1201 loss_recon=0.1201 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9606 corrupt_frac=1.0000 acc_corrupt=0.9606 loss_corrupt=0.1201 wrong_frac=0.7908 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9292 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.1029 out_g_norm=0.1619 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0543 init_gold_top10=0.2187 init_gold_top100=0.2915 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=4.4s lr=2.000000e-03 loss=0.1151 loss_recon=0.1151 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9627 corrupt_frac=1.0000 acc_corrupt=0.9627 loss_corrupt=0.1151 wrong_frac=0.7917 init_acc_corrupt=0.2067 acc_corrupt_t_0p0_0p2=0.9332 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.1280 out_g_norm=0.1673 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0847 init_gold_top10=0.2053 init_gold_top100=0.2800 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=3.7s lr=2.000000e-03 loss=0.1195 loss_recon=0.1195 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9618 corrupt_frac=1.0000 acc_corrupt=0.9618 loss_corrupt=0.1195 wrong_frac=0.7925 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.9315 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=12.1464 out_g_norm=0.1466 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1061 init_gold_top10=0.2043 init_gold_top100=0.2783 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=3.7s lr=2.000000e-03 loss=0.1005 loss_recon=0.1005 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9673 corrupt_frac=1.0000 acc_corrupt=0.9673 loss_corrupt=0.1005 wrong_frac=0.7884 init_acc_corrupt=0.2102 acc_corrupt_t_0p0_0p2=0.9405 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.1477 out_g_norm=0.1316 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0869 init_gold_top10=0.2224 init_gold_top100=0.2981 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=3.7s lr=2.000000e-03 loss=0.1115 loss_recon=0.1115 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9634 corrupt_frac=1.0000 acc_corrupt=0.9634 loss_corrupt=0.1115 wrong_frac=0.7927 init_acc_corrupt=0.2056 acc_corrupt_t_0p0_0p2=0.9349 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=12.1573 out_g_norm=0.1343 loss_all=0.0942 init_gold_top10=0.2197 init_gold_top100=0.2939 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=3.7s lr=2.000000e-03 loss=0.1057 loss_recon=0.1057 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9654 corrupt_frac=1.0000 acc_corrupt=0.9654 loss_corrupt=0.1057 wrong_frac=0.7904 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.9376 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=12.1754 out_g_norm=0.1418 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1002 init_gold_top10=0.2225 init_gold_top100=0.2952 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=3.7s lr=2.000000e-03 loss=0.0998 loss_recon=0.0998 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9678 corrupt_frac=1.0000 acc_corrupt=0.9678 loss_corrupt=0.0998 wrong_frac=0.7910 init_acc_corrupt=0.2074 acc_corrupt_t_0p0_0p2=0.9426 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.1861 out_g_norm=0.1271 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0094 loss_all=0.0707 init_gold_top10=0.2072 init_gold_top100=0.2815 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=3.7s lr=2.000000e-03 loss=0.0970 loss_recon=0.0970 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9686 corrupt_frac=1.0000 acc_corrupt=0.9686 loss_corrupt=0.0970 wrong_frac=0.7907 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9435 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=12.1979 out_g_norm=0.1102 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0570 init_gold_top10=0.2117 init_gold_top100=0.2832 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=3.7s lr=2.000000e-03 loss=0.0949 loss_recon=0.0949 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9698 corrupt_frac=1.0000 acc_corrupt=0.9698 loss_corrupt=0.0949 wrong_frac=0.7897 init_acc_corrupt=0.2086 acc_corrupt_t_0p0_0p2=0.9453 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=12.2015 out_g_norm=0.1196 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0685 init_gold_top10=0.2208 init_gold_top100=0.2931 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=3.7s lr=2.000000e-03 loss=0.0920 loss_recon=0.0920 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9697 corrupt_frac=1.0000 acc_corrupt=0.9697 loss_corrupt=0.0920 wrong_frac=0.7906 init_acc_corrupt=0.2078 acc_corrupt_t_0p0_0p2=0.9458 corrupt_frac_t_0p0_0p2=0.5584 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0801 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2185 out_g_norm=0.1084 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0760 init_gold_top10=0.2190 init_gold_top100=0.2921 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=3.7s lr=2.000000e-03 loss=0.0899 loss_recon=0.0899 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9708 corrupt_frac=1.0000 acc_corrupt=0.9708 loss_corrupt=0.0899 wrong_frac=0.7908 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9476 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.2416 out_g_norm=0.1068 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0347 init_gold_top10=0.2187 init_gold_top100=0.2915 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=4.4s lr=2.000000e-03 loss=0.0858 loss_recon=0.0858 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9728 corrupt_frac=1.0000 acc_corrupt=0.9728 loss_corrupt=0.0858 wrong_frac=0.7917 init_acc_corrupt=0.2067 acc_corrupt_t_0p0_0p2=0.9514 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2445 out_g_norm=0.1039 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0670 init_gold_top10=0.2053 init_gold_top100=0.2800 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=3.7s lr=2.000000e-03 loss=0.0896 loss_recon=0.0896 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9710 corrupt_frac=1.0000 acc_corrupt=0.9710 loss_corrupt=0.0896 wrong_frac=0.7925 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.9480 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=12.2565 out_g_norm=0.1005 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0698 init_gold_top10=0.2043 init_gold_top100=0.2783 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=3.7s lr=2.000000e-03 loss=0.0751 loss_recon=0.0751 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9757 corrupt_frac=1.0000 acc_corrupt=0.9757 loss_corrupt=0.0751 wrong_frac=0.7884 init_acc_corrupt=0.2102 acc_corrupt_t_0p0_0p2=0.9558 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.2656 out_g_norm=0.0997 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0877 init_gold_top10=0.2224 init_gold_top100=0.2981 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=3.7s lr=2.000000e-03 loss=0.0874 loss_recon=0.0874 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9713 corrupt_frac=1.0000 acc_corrupt=0.9713 loss_corrupt=0.0874 wrong_frac=0.7927 init_acc_corrupt=0.2056 acc_corrupt_t_0p0_0p2=0.9490 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=12.2718 out_g_norm=0.0918 loss_all=0.0652 init_gold_top10=0.2197 init_gold_top100=0.2939 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=3.7s lr=2.000000e-03 loss=0.0786 loss_recon=0.0786 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9741 corrupt_frac=1.0000 acc_corrupt=0.9741 loss_corrupt=0.0786 wrong_frac=0.7904 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.9533 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=12.2720 out_g_norm=0.0895 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0935 init_gold_top10=0.2225 init_gold_top100=0.2952 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=3.7s lr=2.000000e-03 loss=0.0763 loss_recon=0.0763 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9753 corrupt_frac=1.0000 acc_corrupt=0.9753 loss_corrupt=0.0763 wrong_frac=0.7910 init_acc_corrupt=0.2074 acc_corrupt_t_0p0_0p2=0.9560 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.2727 out_g_norm=0.0911 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0094 loss_all=0.0643 init_gold_top10=0.2072 init_gold_top100=0.2815 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=3.7s lr=2.000000e-03 loss=0.0781 loss_recon=0.0781 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9748 corrupt_frac=1.0000 acc_corrupt=0.9748 loss_corrupt=0.0781 wrong_frac=0.7907 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9546 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=12.2705 out_g_norm=0.0846 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0330 init_gold_top10=0.2117 init_gold_top100=0.2832 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=3.7s lr=2.000000e-03 loss=0.0692 loss_recon=0.0692 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9780 corrupt_frac=1.0000 acc_corrupt=0.9780 loss_corrupt=0.0692 wrong_frac=0.7897 init_acc_corrupt=0.2086 acc_corrupt_t_0p0_0p2=0.9602 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=12.2632 out_g_norm=0.0868 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0457 init_gold_top10=0.2208 init_gold_top100=0.2931 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=3.7s lr=2.000000e-03 loss=0.0706 loss_recon=0.0706 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9766 corrupt_frac=1.0000 acc_corrupt=0.9766 loss_corrupt=0.0706 wrong_frac=0.7906 init_acc_corrupt=0.2078 acc_corrupt_t_0p0_0p2=0.9581 corrupt_frac_t_0p0_0p2=0.5584 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0801 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2531 out_g_norm=0.0820 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0530 init_gold_top10=0.2190 init_gold_top100=0.2921 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=3.7s lr=2.000000e-03 loss=0.0658 loss_recon=0.0658 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9795 corrupt_frac=1.0000 acc_corrupt=0.9795 loss_corrupt=0.0658 wrong_frac=0.7908 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9631 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.2590 out_g_norm=0.0768 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0183 init_gold_top10=0.2187 init_gold_top100=0.2915 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_highC64_4096_20260517_163805/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=4.4s lr=2.000000e-03 loss=0.0738 loss_recon=0.0738 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9772 corrupt_frac=1.0000 acc_corrupt=0.9772 loss_corrupt=0.0738 wrong_frac=0.7917 init_acc_corrupt=0.2067 acc_corrupt_t_0p0_0p2=0.9592 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=12.2596 out_g_norm=0.0802 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1209 init_gold_top10=0.2053 init_gold_top100=0.2800 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=3.7s lr=2.000000e-03 loss=0.0741 loss_recon=0.0741 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9760 corrupt_frac=1.0000 acc_corrupt=0.9760 loss_corrupt=0.0741 wrong_frac=0.7925 init_acc_corrupt=0.2059 acc_corrupt_t_0p0_0p2=0.9570 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=12.2460 out_g_norm=0.0796 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0588 init_gold_top10=0.2043 init_gold_top100=0.2783 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=3.7s lr=2.000000e-03 loss=0.0609 loss_recon=0.0609 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9804 corrupt_frac=1.0000 acc_corrupt=0.9804 loss_corrupt=0.0609 wrong_frac=0.7884 init_acc_corrupt=0.2102 acc_corrupt_t_0p0_0p2=0.9642 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=12.2396 out_g_norm=0.0777 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0815 init_gold_top10=0.2224 init_gold_top100=0.2981 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=3.7s lr=2.000000e-03 loss=0.0665 loss_recon=0.0665 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9787 corrupt_frac=1.0000 acc_corrupt=0.9787 loss_corrupt=0.0665 wrong_frac=0.7927 init_acc_corrupt=0.2056 acc_corrupt_t_0p0_0p2=0.9621 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=12.2321 out_g_norm=0.0750 loss_all=0.0665 init_gold_top10=0.2197 init_gold_top100=0.2939 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=3.9s lr=2.000000e-03 loss=0.0639 loss_recon=0.0639 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9799 corrupt_frac=1.0000 acc_corrupt=0.9799 loss_corrupt=0.0639 wrong_frac=0.7904 init_acc_corrupt=0.2080 acc_corrupt_t_0p0_0p2=0.9637 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=12.2223 out_g_norm=0.0766 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0581 init_gold_top10=0.2225 init_gold_top100=0.2952 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=3.7s lr=2.000000e-03 loss=0.0625 loss_recon=0.0625 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9799 corrupt_frac=1.0000 acc_corrupt=0.9799 loss_corrupt=0.0625 wrong_frac=0.7910 init_acc_corrupt=0.2074 acc_corrupt_t_0p0_0p2=0.9642 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.1981 out_g_norm=0.0736 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0094 loss_all=0.0510 init_gold_top10=0.2072 init_gold_top100=0.2815 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=3.7s lr=2.000000e-03 loss=0.0641 loss_recon=0.0641 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9791 corrupt_frac=1.0000 acc_corrupt=0.9791 loss_corrupt=0.0641 wrong_frac=0.7907 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9624 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=12.1963 out_g_norm=0.0704 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0676 init_gold_top10=0.2117 init_gold_top100=0.2832 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=3.7s lr=2.000000e-03 loss=0.0633 loss_recon=0.0633 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9797 corrupt_frac=1.0000 acc_corrupt=0.9797 loss_corrupt=0.0633 wrong_frac=0.7897 init_acc_corrupt=0.2086 acc_corrupt_t_0p0_0p2=0.9632 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=12.1704 out_g_norm=0.0736 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0173 init_gold_top10=0.2208 init_gold_top100=0.2931 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805.log new file mode 100644 index 0000000000000000000000000000000000000000..946af72f7aa3477ccc56f8a9673a65b1ac0373c2 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805.log @@ -0,0 +1,791 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.05, + "logistic_normal_sigma_max": 0.5, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.5s lr=2.000000e-03 loss=6.6416 loss_recon=6.6416 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1812 corrupt_frac=1.0000 acc_corrupt=0.1812 loss_corrupt=6.6416 wrong_frac=0.7916 init_acc_corrupt=0.2093 acc_corrupt_t_0p0_0p2=0.0970 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.2433 corrupt_frac_t_0p2_0p4=0.3553 acc_corrupt_t_0p4_0p6=0.4524 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.6374 corrupt_frac_t_0p6_0p8=0.0122 out_w_norm=1.2373 out_g_norm=0.9360 acc_corrupt_t_0p8_1p0=0.8027 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.3029 init_gold_top10=0.2177 init_gold_top100=0.2919 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=3.8s lr=2.000000e-03 loss=5.8451 loss_recon=5.8451 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1702 corrupt_frac=1.0000 acc_corrupt=0.1702 loss_corrupt=5.8451 wrong_frac=0.7931 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.1040 corrupt_frac_t_0p0_0p2=0.5635 acc_corrupt_t_0p2_0p4=0.2165 corrupt_frac_t_0p2_0p4=0.3538 acc_corrupt_t_0p4_0p6=0.4006 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.6151 corrupt_frac_t_0p6_0p8=0.0127 out_w_norm=4.0542 out_g_norm=1.2907 acc_corrupt_t_0p8_1p0=0.7878 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.5064 init_gold_top10=0.1975 init_gold_top100=0.2742 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=3.8s lr=2.000000e-03 loss=5.0754 loss_recon=5.0754 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1945 corrupt_frac=1.0000 acc_corrupt=0.1945 loss_corrupt=5.0754 wrong_frac=0.7916 init_acc_corrupt=0.2092 acc_corrupt_t_0p0_0p2=0.1171 corrupt_frac_t_0p0_0p2=0.5532 acc_corrupt_t_0p2_0p4=0.2511 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=0.4385 corrupt_frac_t_0p4_0p6=0.0771 acc_corrupt_t_0p6_0p8=0.6358 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=6.6304 out_g_norm=0.7911 loss_all=4.7495 init_gold_top10=0.2067 init_gold_top100=0.2811 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=3.8s lr=2.000000e-03 loss=4.2417 loss_recon=4.2417 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2396 corrupt_frac=1.0000 acc_corrupt=0.2396 loss_corrupt=4.2417 wrong_frac=0.7924 init_acc_corrupt=0.2084 acc_corrupt_t_0p0_0p2=0.1509 corrupt_frac_t_0p0_0p2=0.5611 acc_corrupt_t_0p2_0p4=0.3167 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.4920 corrupt_frac_t_0p4_0p6=0.0734 out_w_norm=8.3769 out_g_norm=0.4660 acc_corrupt_t_0p6_0p8=0.6663 corrupt_frac_t_0p6_0p8=0.0122 loss_all=3.9329 init_gold_top10=0.1958 init_gold_top100=0.2700 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=3.8s lr=2.000000e-03 loss=3.2604 loss_recon=3.2604 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2686 corrupt_frac=1.0000 acc_corrupt=0.2686 loss_corrupt=3.2604 wrong_frac=0.7902 init_acc_corrupt=0.2106 acc_corrupt_t_0p0_0p2=0.1795 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=0.3418 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=0.5201 corrupt_frac_t_0p4_0p6=0.0798 out_w_norm=9.7739 out_g_norm=0.4141 acc_corrupt_t_0p6_0p8=0.6828 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=0.8711 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.7174 init_gold_top10=0.2355 init_gold_top100=0.3061 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.8s lr=2.000000e-03 loss=2.2324 loss_recon=2.2324 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3783 corrupt_frac=1.0000 acc_corrupt=0.3783 loss_corrupt=2.2324 wrong_frac=0.7920 init_acc_corrupt=0.2088 acc_corrupt_t_0p0_0p2=0.2666 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.4840 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=0.6511 corrupt_frac_t_0p4_0p6=0.0777 acc_corrupt_t_0p6_0p8=0.7795 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=10.5161 out_g_norm=0.4345 acc_corrupt_t_0p8_1p0=0.8880 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.7136 init_gold_top10=0.2176 init_gold_top100=0.2910 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.8s lr=2.000000e-03 loss=1.1924 loss_recon=1.1924 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6467 corrupt_frac=1.0000 acc_corrupt=0.6467 loss_corrupt=1.1924 wrong_frac=0.7910 init_acc_corrupt=0.2098 acc_corrupt_t_0p0_0p2=0.5041 corrupt_frac_t_0p0_0p2=0.5582 acc_corrupt_t_0p2_0p4=0.8081 corrupt_frac_t_0p2_0p4=0.3555 acc_corrupt_t_0p4_0p6=0.8997 corrupt_frac_t_0p4_0p6=0.0766 acc_corrupt_t_0p6_0p8=0.9358 corrupt_frac_t_0p6_0p8=0.0144 out_w_norm=11.0276 out_g_norm=0.5232 acc_corrupt_t_0p8_1p0=0.9854 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7009 init_gold_top10=0.2223 init_gold_top100=0.2966 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.8s lr=2.000000e-03 loss=0.4993 loss_recon=0.4993 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8635 corrupt_frac=1.0000 acc_corrupt=0.8635 loss_corrupt=0.4993 wrong_frac=0.7890 init_acc_corrupt=0.2118 acc_corrupt_t_0p0_0p2=0.7646 corrupt_frac_t_0p0_0p2=0.5512 acc_corrupt_t_0p2_0p4=0.9823 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.9957 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.4562 out_g_norm=0.4135 acc_corrupt_t_0p6_0p8=0.9968 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3302 init_gold_top10=0.2302 init_gold_top100=0.3015 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.8s lr=2.000000e-03 loss=0.2580 loss_recon=0.2580 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9303 corrupt_frac=1.0000 acc_corrupt=0.9303 loss_corrupt=0.2580 wrong_frac=0.7914 init_acc_corrupt=0.2094 acc_corrupt_t_0p0_0p2=0.8760 corrupt_frac_t_0p0_0p2=0.5583 acc_corrupt_t_0p2_0p4=0.9987 corrupt_frac_t_0p2_0p4=0.3557 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0769 out_w_norm=11.7902 out_g_norm=0.3034 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2191 init_gold_top10=0.1997 init_gold_top100=0.2768 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.8s lr=2.000000e-03 loss=0.1393 loss_recon=0.1393 loss_meanflow=0.0000 mean_model_t=0.2113 mean_corrupt_t=0.2113 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9612 corrupt_frac=1.0000 acc_corrupt=0.9612 loss_corrupt=0.1393 wrong_frac=0.7887 init_acc_corrupt=0.2122 acc_corrupt_t_0p0_0p2=0.9298 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3594 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=11.9794 out_g_norm=0.2449 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0803 init_gold_top10=0.2300 init_gold_top100=0.3030 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.05, + "logistic_normal_sigma_max": 0.5, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=4.5s lr=2.000000e-03 loss=0.1004 loss_recon=0.1004 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9707 corrupt_frac=1.0000 acc_corrupt=0.9707 loss_corrupt=0.1004 wrong_frac=0.7916 init_acc_corrupt=0.2093 acc_corrupt_t_0p0_0p2=0.9476 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3553 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0122 out_w_norm=12.0726 out_g_norm=0.1836 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0791 init_gold_top10=0.2177 init_gold_top100=0.2919 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=3.8s lr=2.000000e-03 loss=0.0856 loss_recon=0.0856 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9749 corrupt_frac=1.0000 acc_corrupt=0.9749 loss_corrupt=0.0856 wrong_frac=0.7931 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9556 corrupt_frac_t_0p0_0p2=0.5635 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3538 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0127 out_w_norm=12.1000 out_g_norm=0.1557 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0377 init_gold_top10=0.1975 init_gold_top100=0.2742 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=3.8s lr=2.000000e-03 loss=0.0679 loss_recon=0.0679 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9793 corrupt_frac=1.0000 acc_corrupt=0.9793 loss_corrupt=0.0679 wrong_frac=0.7916 init_acc_corrupt=0.2092 acc_corrupt_t_0p0_0p2=0.9626 corrupt_frac_t_0p0_0p2=0.5532 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0771 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.0992 out_g_norm=0.1191 loss_all=0.0899 init_gold_top10=0.2067 init_gold_top100=0.2811 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=3.8s lr=2.000000e-03 loss=0.0679 loss_recon=0.0679 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9794 corrupt_frac=1.0000 acc_corrupt=0.9794 loss_corrupt=0.0679 wrong_frac=0.7924 init_acc_corrupt=0.2084 acc_corrupt_t_0p0_0p2=0.9633 corrupt_frac_t_0p0_0p2=0.5611 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 out_w_norm=12.0845 out_g_norm=0.1117 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0122 loss_all=0.0877 init_gold_top10=0.1958 init_gold_top100=0.2700 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=3.8s lr=2.000000e-03 loss=0.0575 loss_recon=0.0575 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9828 corrupt_frac=1.0000 acc_corrupt=0.9828 loss_corrupt=0.0575 wrong_frac=0.7902 init_acc_corrupt=0.2106 acc_corrupt_t_0p0_0p2=0.9692 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0798 out_w_norm=12.0797 out_g_norm=0.1056 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0339 init_gold_top10=0.2355 init_gold_top100=0.3061 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=3.8s lr=2.000000e-03 loss=0.0564 loss_recon=0.0564 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9829 corrupt_frac=1.0000 acc_corrupt=0.9829 loss_corrupt=0.0564 wrong_frac=0.7920 init_acc_corrupt=0.2088 acc_corrupt_t_0p0_0p2=0.9693 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0777 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.0764 out_g_norm=0.0942 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1145 init_gold_top10=0.2176 init_gold_top100=0.2910 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=3.8s lr=2.000000e-03 loss=0.0459 loss_recon=0.0459 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9860 corrupt_frac=1.0000 acc_corrupt=0.9860 loss_corrupt=0.0459 wrong_frac=0.7910 init_acc_corrupt=0.2098 acc_corrupt_t_0p0_0p2=0.9750 corrupt_frac_t_0p0_0p2=0.5582 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3555 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0766 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0144 out_w_norm=12.0780 out_g_norm=0.0834 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0271 init_gold_top10=0.2223 init_gold_top100=0.2966 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=3.8s lr=2.000000e-03 loss=0.0491 loss_recon=0.0491 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9850 corrupt_frac=1.0000 acc_corrupt=0.9850 loss_corrupt=0.0491 wrong_frac=0.7890 init_acc_corrupt=0.2118 acc_corrupt_t_0p0_0p2=0.9729 corrupt_frac_t_0p0_0p2=0.5512 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.0707 out_g_norm=0.0727 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0427 init_gold_top10=0.2302 init_gold_top100=0.3015 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=3.8s lr=2.000000e-03 loss=0.0457 loss_recon=0.0457 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9862 corrupt_frac=1.0000 acc_corrupt=0.9862 loss_corrupt=0.0457 wrong_frac=0.7914 init_acc_corrupt=0.2094 acc_corrupt_t_0p0_0p2=0.9752 corrupt_frac_t_0p0_0p2=0.5583 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3557 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0769 out_w_norm=12.0792 out_g_norm=0.0672 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0452 init_gold_top10=0.1997 init_gold_top100=0.2768 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=3.8s lr=2.000000e-03 loss=0.0360 loss_recon=0.0360 loss_meanflow=0.0000 mean_model_t=0.2113 mean_corrupt_t=0.2113 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9890 corrupt_frac=1.0000 acc_corrupt=0.9890 loss_corrupt=0.0360 wrong_frac=0.7887 init_acc_corrupt=0.2122 acc_corrupt_t_0p0_0p2=0.9800 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3594 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.0823 out_g_norm=0.0653 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0224 init_gold_top10=0.2300 init_gold_top100=0.3030 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.05, + "logistic_normal_sigma_max": 0.5, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=4.5s lr=2.000000e-03 loss=0.0377 loss_recon=0.0377 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9883 corrupt_frac=1.0000 acc_corrupt=0.9883 loss_corrupt=0.0377 wrong_frac=0.7916 init_acc_corrupt=0.2093 acc_corrupt_t_0p0_0p2=0.9791 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3553 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 out_w_norm=12.0951 out_g_norm=0.0622 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0344 init_gold_top10=0.2177 init_gold_top100=0.2919 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=4.0s lr=2.000000e-03 loss=0.0402 loss_recon=0.0402 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9881 corrupt_frac=1.0000 acc_corrupt=0.9881 loss_corrupt=0.0402 wrong_frac=0.7931 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9788 corrupt_frac_t_0p0_0p2=0.5635 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3538 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0127 out_w_norm=12.1114 out_g_norm=0.0591 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0402 init_gold_top10=0.1975 init_gold_top100=0.2742 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=3.9s lr=2.000000e-03 loss=0.0340 loss_recon=0.0340 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9892 corrupt_frac=1.0000 acc_corrupt=0.9892 loss_corrupt=0.0340 wrong_frac=0.7916 init_acc_corrupt=0.2092 acc_corrupt_t_0p0_0p2=0.9805 corrupt_frac_t_0p0_0p2=0.5532 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0771 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.1281 out_g_norm=0.0558 loss_all=0.0586 init_gold_top10=0.2067 init_gold_top100=0.2811 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=3.9s lr=2.000000e-03 loss=0.0344 loss_recon=0.0344 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9894 corrupt_frac=1.0000 acc_corrupt=0.9894 loss_corrupt=0.0344 wrong_frac=0.7924 init_acc_corrupt=0.2084 acc_corrupt_t_0p0_0p2=0.9811 corrupt_frac_t_0p0_0p2=0.5611 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 out_w_norm=12.1489 out_g_norm=0.0537 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 loss_all=0.0615 init_gold_top10=0.1958 init_gold_top100=0.2700 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=3.9s lr=2.000000e-03 loss=0.0312 loss_recon=0.0312 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9905 corrupt_frac=1.0000 acc_corrupt=0.9905 loss_corrupt=0.0312 wrong_frac=0.7902 init_acc_corrupt=0.2106 acc_corrupt_t_0p0_0p2=0.9830 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0798 out_w_norm=12.1605 out_g_norm=0.0507 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0138 init_gold_top10=0.2355 init_gold_top100=0.3061 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=3.9s lr=2.000000e-03 loss=0.0313 loss_recon=0.0313 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9904 corrupt_frac=1.0000 acc_corrupt=0.9904 loss_corrupt=0.0313 wrong_frac=0.7920 init_acc_corrupt=0.2088 acc_corrupt_t_0p0_0p2=0.9828 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0777 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.1766 out_g_norm=0.0494 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0815 init_gold_top10=0.2176 init_gold_top100=0.2910 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=3.9s lr=2.000000e-03 loss=0.0249 loss_recon=0.0249 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9922 corrupt_frac=1.0000 acc_corrupt=0.9922 loss_corrupt=0.0249 wrong_frac=0.7910 init_acc_corrupt=0.2098 acc_corrupt_t_0p0_0p2=0.9860 corrupt_frac_t_0p0_0p2=0.5582 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3555 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0766 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0144 out_w_norm=12.1834 out_g_norm=0.0451 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0019 init_gold_top10=0.2223 init_gold_top100=0.2966 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=3.9s lr=2.000000e-03 loss=0.0284 loss_recon=0.0284 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9912 corrupt_frac=1.0000 acc_corrupt=0.9912 loss_corrupt=0.0284 wrong_frac=0.7890 init_acc_corrupt=0.2118 acc_corrupt_t_0p0_0p2=0.9841 corrupt_frac_t_0p0_0p2=0.5512 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.2032 out_g_norm=0.0412 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0442 init_gold_top10=0.2302 init_gold_top100=0.3015 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=3.9s lr=2.000000e-03 loss=0.0276 loss_recon=0.0276 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9916 corrupt_frac=1.0000 acc_corrupt=0.9916 loss_corrupt=0.0276 wrong_frac=0.7914 init_acc_corrupt=0.2094 acc_corrupt_t_0p0_0p2=0.9849 corrupt_frac_t_0p0_0p2=0.5583 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3557 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0769 out_w_norm=12.2219 out_g_norm=0.0415 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0432 init_gold_top10=0.1997 init_gold_top100=0.2768 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=3.9s lr=2.000000e-03 loss=0.0224 loss_recon=0.0224 loss_meanflow=0.0000 mean_model_t=0.2113 mean_corrupt_t=0.2113 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9929 corrupt_frac=1.0000 acc_corrupt=0.9929 loss_corrupt=0.0224 wrong_frac=0.7887 init_acc_corrupt=0.2122 acc_corrupt_t_0p0_0p2=0.9872 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3594 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.2261 out_g_norm=0.0409 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0203 init_gold_top10=0.2300 init_gold_top100=0.3030 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "logistic_normal_linear_mean", + "logistic_normal_sigma_min": 0.05, + "logistic_normal_sigma_max": 0.5, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_logistic_sig0p05_0p5_20260517_163805/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=4.4s lr=2.000000e-03 loss=0.0224 loss_recon=0.0224 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9929 corrupt_frac=1.0000 acc_corrupt=0.9929 loss_corrupt=0.0224 wrong_frac=0.7916 init_acc_corrupt=0.2093 acc_corrupt_t_0p0_0p2=0.9873 corrupt_frac_t_0p0_0p2=0.5587 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3553 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0770 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0122 out_w_norm=12.2369 out_g_norm=0.0382 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0017 init_gold_top10=0.2177 init_gold_top100=0.2919 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=4.0s lr=2.000000e-03 loss=0.0248 loss_recon=0.0248 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9927 corrupt_frac=1.0000 acc_corrupt=0.9927 loss_corrupt=0.0248 wrong_frac=0.7931 init_acc_corrupt=0.2077 acc_corrupt_t_0p0_0p2=0.9870 corrupt_frac_t_0p0_0p2=0.5635 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3538 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0127 out_w_norm=12.2663 out_g_norm=0.0392 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0048 init_gold_top10=0.1975 init_gold_top100=0.2742 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=4.0s lr=2.000000e-03 loss=0.0220 loss_recon=0.0220 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9929 corrupt_frac=1.0000 acc_corrupt=0.9929 loss_corrupt=0.0220 wrong_frac=0.7916 init_acc_corrupt=0.2092 acc_corrupt_t_0p0_0p2=0.9872 corrupt_frac_t_0p0_0p2=0.5532 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0771 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=12.2826 out_g_norm=0.0395 loss_all=0.0236 init_gold_top10=0.2067 init_gold_top100=0.2811 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=3.9s lr=2.000000e-03 loss=0.0221 loss_recon=0.0221 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9931 corrupt_frac=1.0000 acc_corrupt=0.9931 loss_corrupt=0.0221 wrong_frac=0.7924 init_acc_corrupt=0.2084 acc_corrupt_t_0p0_0p2=0.9878 corrupt_frac_t_0p0_0p2=0.5611 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 out_w_norm=12.2881 out_g_norm=0.0364 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0122 loss_all=0.0122 init_gold_top10=0.1958 init_gold_top100=0.2700 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=3.9s lr=2.000000e-03 loss=0.0195 loss_recon=0.0195 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9940 corrupt_frac=1.0000 acc_corrupt=0.9940 loss_corrupt=0.0195 wrong_frac=0.7902 init_acc_corrupt=0.2106 acc_corrupt_t_0p0_0p2=0.9892 corrupt_frac_t_0p0_0p2=0.5567 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3548 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0798 out_w_norm=12.2898 out_g_norm=0.0367 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0037 init_gold_top10=0.2355 init_gold_top100=0.3061 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=3.9s lr=2.000000e-03 loss=0.0201 loss_recon=0.0201 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9938 corrupt_frac=1.0000 acc_corrupt=0.9938 loss_corrupt=0.0201 wrong_frac=0.7920 init_acc_corrupt=0.2088 acc_corrupt_t_0p0_0p2=0.9889 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3566 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0777 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.2942 out_g_norm=0.0362 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0312 init_gold_top10=0.2176 init_gold_top100=0.2910 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=3.9s lr=2.000000e-03 loss=0.0160 loss_recon=0.0160 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9951 corrupt_frac=1.0000 acc_corrupt=0.9951 loss_corrupt=0.0160 wrong_frac=0.7910 init_acc_corrupt=0.2098 acc_corrupt_t_0p0_0p2=0.9912 corrupt_frac_t_0p0_0p2=0.5582 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3555 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0766 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0144 out_w_norm=12.2805 out_g_norm=0.0321 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0018 init_gold_top10=0.2223 init_gold_top100=0.2966 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=3.9s lr=2.000000e-03 loss=0.0190 loss_recon=0.0190 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9940 corrupt_frac=1.0000 acc_corrupt=0.9940 loss_corrupt=0.0190 wrong_frac=0.7890 init_acc_corrupt=0.2118 acc_corrupt_t_0p0_0p2=0.9891 corrupt_frac_t_0p0_0p2=0.5512 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.2694 out_g_norm=0.0318 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0473 init_gold_top10=0.2302 init_gold_top100=0.3015 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=3.9s lr=2.000000e-03 loss=0.0213 loss_recon=0.0213 loss_meanflow=0.0000 mean_model_t=0.2088 mean_corrupt_t=0.2088 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9935 corrupt_frac=1.0000 acc_corrupt=0.9935 loss_corrupt=0.0213 wrong_frac=0.7914 init_acc_corrupt=0.2094 acc_corrupt_t_0p0_0p2=0.9883 corrupt_frac_t_0p0_0p2=0.5583 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3557 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0769 out_w_norm=12.2660 out_g_norm=0.0325 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0110 init_gold_top10=0.1997 init_gold_top100=0.2768 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=3.9s lr=2.000000e-03 loss=0.0161 loss_recon=0.0161 loss_meanflow=0.0000 mean_model_t=0.2113 mean_corrupt_t=0.2113 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9951 corrupt_frac=1.0000 acc_corrupt=0.9951 loss_corrupt=0.0161 wrong_frac=0.7887 init_acc_corrupt=0.2122 acc_corrupt_t_0p0_0p2=0.9910 corrupt_frac_t_0p0_0p2=0.5509 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3594 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0805 out_w_norm=12.2661 out_g_norm=0.0345 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0125 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0059 init_gold_top10=0.2300 init_gold_top100=0.3030 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805.log new file mode 100644 index 0000000000000000000000000000000000000000..5093a5aa7a4295f49ae724d6f2b7724613e36e01 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805.log @@ -0,0 +1,634 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.5s lr=2.000000e-03 loss=6.7171 loss_recon=6.7171 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0942 corrupt_frac=1.0000 acc_corrupt=0.0942 loss_corrupt=6.7171 wrong_frac=0.7922 init_acc_corrupt=0.1148 acc_corrupt_t_0p0_0p2=0.0455 corrupt_frac_t_0p0_0p2=0.5620 acc_corrupt_t_0p2_0p4=0.1254 corrupt_frac_t_0p2_0p4=0.3533 acc_corrupt_t_0p4_0p6=0.2713 corrupt_frac_t_0p4_0p6=0.0755 acc_corrupt_t_0p6_0p8=0.4107 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=1.0887 out_g_norm=1.0092 acc_corrupt_t_0p8_1p0=0.8945 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.4630 init_gold_top10=0.2117 init_gold_top100=0.4247 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=3.8s lr=2.000000e-03 loss=6.1879 loss_recon=6.1879 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1016 corrupt_frac=1.0000 acc_corrupt=0.1016 loss_corrupt=6.1879 wrong_frac=0.7916 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.0495 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.1388 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.2810 corrupt_frac_t_0p4_0p6=0.0744 acc_corrupt_t_0p6_0p8=0.3941 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.4678 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=3.3136 out_g_norm=1.4051 loss_all=5.9306 init_gold_top10=0.2016 init_gold_top100=0.4192 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=3.8s lr=2.000000e-03 loss=5.7129 loss_recon=5.7129 loss_meanflow=0.0000 mean_model_t=0.2105 mean_corrupt_t=0.2105 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1216 corrupt_frac=1.0000 acc_corrupt=0.1216 loss_corrupt=5.7129 wrong_frac=0.7894 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.0537 corrupt_frac_t_0p0_0p2=0.5542 acc_corrupt_t_0p2_0p4=0.1615 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.3620 corrupt_frac_t_0p4_0p6=0.0813 acc_corrupt_t_0p6_0p8=0.5778 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=5.3362 out_g_norm=0.6855 acc_corrupt_t_0p8_1p0=0.5938 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.4734 init_gold_top10=0.2133 init_gold_top100=0.4237 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=3.8s lr=2.000000e-03 loss=5.3493 loss_recon=5.3493 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1453 corrupt_frac=1.0000 acc_corrupt=0.1453 loss_corrupt=5.3493 wrong_frac=0.7915 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.0574 corrupt_frac_t_0p0_0p2=0.5597 acc_corrupt_t_0p2_0p4=0.2021 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.4675 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.6597 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=7.0676 out_g_norm=0.3587 acc_corrupt_t_0p8_1p0=0.7930 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.0547 init_gold_top10=0.2146 init_gold_top100=0.4290 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=3.8s lr=2.000000e-03 loss=4.9052 loss_recon=4.9052 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1578 corrupt_frac=1.0000 acc_corrupt=0.1578 loss_corrupt=4.9052 wrong_frac=0.7914 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.0608 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.2300 corrupt_frac_t_0p2_0p4=0.3550 acc_corrupt_t_0p4_0p6=0.4843 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.6602 corrupt_frac_t_0p6_0p8=0.0143 out_w_norm=8.6464 out_g_norm=0.4603 loss_all=4.6661 init_gold_top10=0.1906 init_gold_top100=0.4129 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.8s lr=2.000000e-03 loss=4.3633 loss_recon=4.3633 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1713 corrupt_frac=1.0000 acc_corrupt=0.1713 loss_corrupt=4.3633 wrong_frac=0.7917 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.0670 corrupt_frac_t_0p0_0p2=0.5568 acc_corrupt_t_0p2_0p4=0.2545 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.4927 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.6751 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=9.6855 out_g_norm=0.4965 loss_all=4.2113 init_gold_top10=0.1772 init_gold_top100=0.4130 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.8s lr=2.000000e-03 loss=3.8335 loss_recon=3.8335 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1847 corrupt_frac=1.0000 acc_corrupt=0.1847 loss_corrupt=3.8335 wrong_frac=0.7925 init_acc_corrupt=0.1147 acc_corrupt_t_0p0_0p2=0.0766 corrupt_frac_t_0p0_0p2=0.5607 acc_corrupt_t_0p2_0p4=0.2754 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.5085 corrupt_frac_t_0p4_0p6=0.0739 acc_corrupt_t_0p6_0p8=0.6886 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=10.2232 out_g_norm=0.6982 loss_all=3.5264 init_gold_top10=0.2137 init_gold_top100=0.4244 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.8s lr=2.000000e-03 loss=3.3860 loss_recon=3.3860 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2059 corrupt_frac=1.0000 acc_corrupt=0.2059 loss_corrupt=3.3860 wrong_frac=0.7904 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.0875 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.3057 corrupt_frac_t_0p2_0p4=0.3649 acc_corrupt_t_0p4_0p6=0.5299 corrupt_frac_t_0p4_0p6=0.0765 out_w_norm=10.5877 out_g_norm=0.9937 acc_corrupt_t_0p6_0p8=0.6989 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=0.8398 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.1965 init_gold_top10=0.1935 init_gold_top100=0.4118 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.8s lr=2.000000e-03 loss=3.0038 loss_recon=3.0038 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2384 corrupt_frac=1.0000 acc_corrupt=0.2384 loss_corrupt=3.0038 wrong_frac=0.7901 init_acc_corrupt=0.1169 acc_corrupt_t_0p0_0p2=0.1048 corrupt_frac_t_0p0_0p2=0.5513 acc_corrupt_t_0p2_0p4=0.3572 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.5844 corrupt_frac_t_0p4_0p6=0.0755 out_w_norm=10.8302 out_g_norm=1.2164 acc_corrupt_t_0p6_0p8=0.7286 corrupt_frac_t_0p6_0p8=0.0127 acc_corrupt_t_0p8_1p0=0.8242 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.9094 init_gold_top10=0.1910 init_gold_top100=0.4159 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.8s lr=2.000000e-03 loss=2.6625 loss_recon=2.6625 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2880 corrupt_frac=1.0000 acc_corrupt=0.2880 loss_corrupt=2.6625 wrong_frac=0.7897 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.1255 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.4443 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.6811 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p8_1p0=0.8555 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=10.9865 out_g_norm=1.5042 acc_corrupt_t_0p6_0p8=0.7904 corrupt_frac_t_0p6_0p8=0.0133 loss_all=2.5068 init_gold_top10=0.1951 init_gold_top100=0.4188 +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run + self._initialize_workers(self._worker_group) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers + self._rendezvous(worker_group) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 500, in _rendezvous + rdzv_info = spec.rdzv_handler.next_rendezvous() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/rendezvous/static_tcp_rendezvous.py", line 67, in next_rendezvous + self._store = TCPStore( # type: ignore[call-arg] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +RuntimeError: The server socket has failed to listen on any local network address. port: 32976, useIpv6: 0, code: -98, name: EADDRINUSE, message: address already in use +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=4.8s lr=2.000000e-03 loss=2.2796 loss_recon=2.2796 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3636 corrupt_frac=1.0000 acc_corrupt=0.3636 loss_corrupt=2.2796 wrong_frac=0.7922 init_acc_corrupt=0.1148 acc_corrupt_t_0p0_0p2=0.1696 corrupt_frac_t_0p0_0p2=0.5620 acc_corrupt_t_0p2_0p4=0.5667 corrupt_frac_t_0p2_0p4=0.3533 acc_corrupt_t_0p4_0p6=0.7971 corrupt_frac_t_0p4_0p6=0.0755 acc_corrupt_t_0p6_0p8=0.8522 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.0831 out_g_norm=1.6545 acc_corrupt_t_0p8_1p0=0.9609 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.9833 init_gold_top10=0.2117 init_gold_top100=0.4247 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=4.1s lr=2.000000e-03 loss=1.8033 loss_recon=1.8033 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4715 corrupt_frac=1.0000 acc_corrupt=0.4715 loss_corrupt=1.8033 wrong_frac=0.7916 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.2501 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.7186 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.8878 corrupt_frac_t_0p4_0p6=0.0744 acc_corrupt_t_0p6_0p8=0.9219 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.9346 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.1814 out_g_norm=1.7198 loss_all=1.5160 init_gold_top10=0.2016 init_gold_top100=0.4192 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=4.1s lr=2.000000e-03 loss=1.4037 loss_recon=1.4037 loss_meanflow=0.0000 mean_model_t=0.2105 mean_corrupt_t=0.2105 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5691 corrupt_frac=1.0000 acc_corrupt=0.5691 loss_corrupt=1.4037 wrong_frac=0.7894 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.3382 corrupt_frac_t_0p0_0p2=0.5542 acc_corrupt_t_0p2_0p4=0.8347 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.9404 corrupt_frac_t_0p4_0p6=0.0813 acc_corrupt_t_0p6_0p8=0.9533 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=11.3091 out_g_norm=1.9176 acc_corrupt_t_0p8_1p0=0.9688 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0970 init_gold_top10=0.2133 init_gold_top100=0.4237 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=4.1s lr=2.000000e-03 loss=1.1179 loss_recon=1.1179 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6413 corrupt_frac=1.0000 acc_corrupt=0.6413 loss_corrupt=1.1179 wrong_frac=0.7915 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.4109 corrupt_frac_t_0p0_0p2=0.5597 acc_corrupt_t_0p2_0p4=0.9243 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.9755 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9771 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=11.4008 out_g_norm=1.8839 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0257 init_gold_top10=0.2146 init_gold_top100=0.4290 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=4.1s lr=2.000000e-03 loss=0.9652 loss_recon=0.9652 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6788 corrupt_frac=1.0000 acc_corrupt=0.6788 loss_corrupt=0.9652 wrong_frac=0.7914 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.4517 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.9651 corrupt_frac_t_0p2_0p4=0.3550 acc_corrupt_t_0p4_0p6=0.9896 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.9916 corrupt_frac_t_0p6_0p8=0.0143 out_w_norm=11.4319 out_g_norm=1.6646 loss_all=0.9362 init_gold_top10=0.1906 init_gold_top100=0.4129 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=4.1s lr=2.000000e-03 loss=0.8711 loss_recon=0.8711 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7041 corrupt_frac=1.0000 acc_corrupt=0.7041 loss_corrupt=0.8711 wrong_frac=0.7917 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.4811 corrupt_frac_t_0p0_0p2=0.5568 acc_corrupt_t_0p2_0p4=0.9818 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9955 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.9946 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.4282 out_g_norm=1.3996 loss_all=0.8841 init_gold_top10=0.1772 init_gold_top100=0.4130 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=4.1s lr=2.000000e-03 loss=0.8025 loss_recon=0.8025 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7223 corrupt_frac=1.0000 acc_corrupt=0.7223 loss_corrupt=0.8025 wrong_frac=0.7925 init_acc_corrupt=0.1147 acc_corrupt_t_0p0_0p2=0.5119 corrupt_frac_t_0p0_0p2=0.5607 acc_corrupt_t_0p2_0p4=0.9893 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9980 corrupt_frac_t_0p4_0p6=0.0739 acc_corrupt_t_0p6_0p8=0.9964 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.4249 out_g_norm=1.2780 loss_all=0.7132 init_gold_top10=0.2137 init_gold_top100=0.4244 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=4.1s lr=2.000000e-03 loss=0.7516 loss_recon=0.7516 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7385 corrupt_frac=1.0000 acc_corrupt=0.7385 loss_corrupt=0.7516 wrong_frac=0.7904 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.5306 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9921 corrupt_frac_t_0p2_0p4=0.3649 acc_corrupt_t_0p4_0p6=0.9986 corrupt_frac_t_0p4_0p6=0.0765 out_w_norm=11.4288 out_g_norm=1.0979 acc_corrupt_t_0p6_0p8=0.9968 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6790 init_gold_top10=0.1935 init_gold_top100=0.4118 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=4.1s lr=2.000000e-03 loss=0.7025 loss_recon=0.7025 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7553 corrupt_frac=1.0000 acc_corrupt=0.7553 loss_corrupt=0.7025 wrong_frac=0.7901 init_acc_corrupt=0.1169 acc_corrupt_t_0p0_0p2=0.5593 corrupt_frac_t_0p0_0p2=0.5513 acc_corrupt_t_0p2_0p4=0.9956 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.0755 out_w_norm=11.4342 out_g_norm=0.9973 acc_corrupt_t_0p6_0p8=0.9980 corrupt_frac_t_0p6_0p8=0.0127 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6958 init_gold_top10=0.1910 init_gold_top100=0.4159 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=4.1s lr=2.000000e-03 loss=0.6663 loss_recon=0.6663 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7647 corrupt_frac=1.0000 acc_corrupt=0.7647 loss_corrupt=0.6663 wrong_frac=0.7897 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.5808 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.9975 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.4442 out_g_norm=0.8627 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0133 loss_all=0.6893 init_gold_top10=0.1951 init_gold_top100=0.4188 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 1.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.5, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_noisegeo_len256_allcorrupt_unigram_shared0p5_20260517_163805/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=4.5s lr=2.000000e-03 loss=0.6586 loss_recon=0.6586 loss_meanflow=0.0000 mean_model_t=0.2078 mean_corrupt_t=0.2078 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7668 corrupt_frac=1.0000 acc_corrupt=0.7668 loss_corrupt=0.6586 wrong_frac=0.7922 init_acc_corrupt=0.1148 acc_corrupt_t_0p0_0p2=0.5865 corrupt_frac_t_0p0_0p2=0.5620 acc_corrupt_t_0p2_0p4=0.9977 corrupt_frac_t_0p2_0p4=0.3533 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0755 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0117 out_w_norm=11.4584 out_g_norm=0.8178 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6532 init_gold_top10=0.2117 init_gold_top100=0.4247 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=3.8s lr=2.000000e-03 loss=0.6295 loss_recon=0.6295 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7760 corrupt_frac=1.0000 acc_corrupt=0.7760 loss_corrupt=0.6295 wrong_frac=0.7916 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.5995 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.9984 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0744 acc_corrupt_t_0p6_0p8=0.9990 corrupt_frac_t_0p6_0p8=0.0126 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.4663 out_g_norm=0.7199 loss_all=0.6324 init_gold_top10=0.2016 init_gold_top100=0.4192 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=3.8s lr=2.000000e-03 loss=0.6050 loss_recon=0.6050 loss_meanflow=0.0000 mean_model_t=0.2105 mean_corrupt_t=0.2105 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7846 corrupt_frac=1.0000 acc_corrupt=0.7846 loss_corrupt=0.6050 wrong_frac=0.7894 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.6123 corrupt_frac_t_0p0_0p2=0.5542 acc_corrupt_t_0p2_0p4=0.9985 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0813 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=11.4696 out_g_norm=0.6746 acc_corrupt_t_0p8_1p0=0.9961 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4587 init_gold_top10=0.2133 init_gold_top100=0.4237 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=3.8s lr=2.000000e-03 loss=0.5927 loss_recon=0.5927 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7908 corrupt_frac=1.0000 acc_corrupt=0.7908 loss_corrupt=0.5927 wrong_frac=0.7915 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.6268 corrupt_frac_t_0p0_0p2=0.5597 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.3559 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0131 out_w_norm=11.4838 out_g_norm=0.6499 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5744 init_gold_top10=0.2146 init_gold_top100=0.4290 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=3.8s lr=2.000000e-03 loss=0.5809 loss_recon=0.5809 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7926 corrupt_frac=1.0000 acc_corrupt=0.7926 loss_corrupt=0.5809 wrong_frac=0.7914 init_acc_corrupt=0.1160 acc_corrupt_t_0p0_0p2=0.6313 corrupt_frac_t_0p0_0p2=0.5616 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.3550 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0741 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0143 out_w_norm=11.5089 out_g_norm=0.5936 loss_all=0.5771 init_gold_top10=0.1906 init_gold_top100=0.4129 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=3.8s lr=2.000000e-03 loss=0.5694 loss_recon=0.5694 loss_meanflow=0.0000 mean_model_t=0.2084 mean_corrupt_t=0.2084 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7986 corrupt_frac=1.0000 acc_corrupt=0.7986 loss_corrupt=0.5694 wrong_frac=0.7917 init_acc_corrupt=0.1154 acc_corrupt_t_0p0_0p2=0.6388 corrupt_frac_t_0p0_0p2=0.5568 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3606 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0742 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.5321 out_g_norm=0.5380 loss_all=0.6365 init_gold_top10=0.1772 init_gold_top100=0.4130 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=3.8s lr=2.000000e-03 loss=0.5522 loss_recon=0.5522 loss_meanflow=0.0000 mean_model_t=0.2079 mean_corrupt_t=0.2079 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8042 corrupt_frac=1.0000 acc_corrupt=0.8042 loss_corrupt=0.5522 wrong_frac=0.7925 init_acc_corrupt=0.1147 acc_corrupt_t_0p0_0p2=0.6515 corrupt_frac_t_0p0_0p2=0.5607 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0739 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0132 out_w_norm=11.5532 out_g_norm=0.5052 loss_all=0.5069 init_gold_top10=0.2137 init_gold_top100=0.4244 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=3.8s lr=2.000000e-03 loss=0.5327 loss_recon=0.5327 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8109 corrupt_frac=1.0000 acc_corrupt=0.8109 loss_corrupt=0.5327 wrong_frac=0.7904 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.6569 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3649 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0765 out_w_norm=11.5678 out_g_norm=0.4714 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0123 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4982 init_gold_top10=0.1935 init_gold_top100=0.4118 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=3.8s lr=2.000000e-03 loss=0.5118 loss_recon=0.5118 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8184 corrupt_frac=1.0000 acc_corrupt=0.8184 loss_corrupt=0.5118 wrong_frac=0.7901 init_acc_corrupt=0.1169 acc_corrupt_t_0p0_0p2=0.6710 corrupt_frac_t_0p0_0p2=0.5513 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0755 out_w_norm=11.5866 out_g_norm=0.4311 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0127 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4894 init_gold_top10=0.1910 init_gold_top100=0.4159 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=3.8s lr=2.000000e-03 loss=0.5007 loss_recon=0.5007 loss_meanflow=0.0000 mean_model_t=0.2101 mean_corrupt_t=0.2101 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8232 corrupt_frac=1.0000 acc_corrupt=0.8232 loss_corrupt=0.5007 wrong_frac=0.7897 init_acc_corrupt=0.1182 acc_corrupt_t_0p0_0p2=0.6841 corrupt_frac_t_0p0_0p2=0.5591 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3526 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0791 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=11.6083 out_g_norm=0.3939 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0133 loss_all=0.5446 init_gold_top10=0.1951 init_gold_top100=0.4188 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused.log new file mode 100644 index 0000000000000000000000000000000000000000..e2df5017f2aa557554e8aa5dc26f388cd06e1f5c --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused.log @@ -0,0 +1,193 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 1.0, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=8.2s lr=2.000000e-03 loss=6.7057 loss_recon=6.7057 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0995 corrupt_frac=1.0000 acc_corrupt=0.0995 loss_corrupt=6.7057 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0488 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1333 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2777 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3853 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0988 out_g_norm=1.0104 loss_all=6.4460 init_gold_top10=0.2110 init_gold_top100=0.5461 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1169 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1051 logit_acc_rollout_kept=0.0000 +step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=7.5s lr=2.000000e-03 loss=6.0920 loss_recon=6.0920 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1116 corrupt_frac=1.0000 acc_corrupt=0.1116 loss_corrupt=6.0920 wrong_frac=0.7892 init_acc_corrupt=0.1190 acc_corrupt_t_0p0_0p2=0.0551 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1512 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2945 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4229 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3334 out_g_norm=1.4060 acc_corrupt_t_0p8_1p0=0.4766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8240 init_gold_top10=0.2049 init_gold_top100=0.5963 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1107 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1096 logit_acc_rollout_kept=0.0000 +step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=7.4s lr=2.000000e-03 loss=5.5560 loss_recon=5.5560 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1210 corrupt_frac=1.0000 acc_corrupt=0.1210 loss_corrupt=5.5560 wrong_frac=0.7935 init_acc_corrupt=0.1153 acc_corrupt_t_0p0_0p2=0.0590 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1694 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3234 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4773 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.2201 out_g_norm=0.7125 acc_corrupt_t_0p8_1p0=0.6380 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.2826 init_gold_top10=0.2209 init_gold_top100=0.6553 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1227 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1303 logit_acc_rollout_kept=0.0000 +step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=7.3s lr=2.000000e-03 loss=4.9781 loss_recon=4.9781 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1483 corrupt_frac=1.0000 acc_corrupt=0.1483 loss_corrupt=4.9781 wrong_frac=0.7917 init_acc_corrupt=0.1181 acc_corrupt_t_0p0_0p2=0.0642 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.2056 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4439 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.9063 out_g_norm=0.4180 acc_corrupt_t_0p6_0p8=0.6502 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7422 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7369 init_gold_top10=0.2113 init_gold_top100=0.7779 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1016 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1530 logit_acc_rollout_kept=0.0000 +step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=7.3s lr=2.000000e-03 loss=4.1805 loss_recon=4.1805 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1890 corrupt_frac=1.0000 acc_corrupt=0.1890 loss_corrupt=4.1805 wrong_frac=0.7928 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.0758 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2839 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5403 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.7033 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8555 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.4044 out_g_norm=0.4606 loss_all=3.6550 init_gold_top10=0.2562 init_gold_top100=0.9090 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1274 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.2250 logit_acc_rollout_kept=0.0000 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024.log new file mode 100644 index 0000000000000000000000000000000000000000..71871edb78de4693144ce17660bcba0a5281b795 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024.log @@ -0,0 +1,397 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024", + "batch_size": 32, + "grad_accum": 4, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=400 elapsed=38.3s lr=2.000000e-03 loss=7.7211 loss_recon=7.7211 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5055 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0955 corrupt_frac=1.0000 acc_corrupt=0.0955 loss_corrupt=7.7211 wrong_frac=0.7908 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.0500 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.1263 corrupt_frac_t_0p2_0p4=0.3604 acc_corrupt_t_0p4_0p6=0.2508 corrupt_frac_t_0p4_0p6=0.0822 acc_corrupt_t_0p6_0p8=0.3566 corrupt_frac_t_0p6_0p8=0.0356 out_w_norm=1.0063 out_g_norm=0.6737 loss_all=7.5189 init_gold_top10=0.1519 init_gold_top100=0.4115 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.0816 init_acc_rollout_kept=0.0578 logit_acc_rollout_applied=0.0869 logit_acc_rollout_kept=0.0730 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=800 elapsed=37.5s lr=2.000000e-03 loss=7.0875 loss_recon=7.0875 loss_meanflow=0.0000 mean_model_t=0.2087 mean_corrupt_t=0.2087 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5035 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1032 corrupt_frac=1.0000 acc_corrupt=0.1032 loss_corrupt=7.0875 wrong_frac=0.7913 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.0556 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.1398 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.2566 corrupt_frac_t_0p4_0p6=0.0807 acc_corrupt_t_0p8_1p0=0.4570 corrupt_frac_t_0p8_1p0=0.0312 out_w_norm=2.8644 out_g_norm=1.0978 acc_corrupt_t_0p6_0p8=0.3368 corrupt_frac_t_0p6_0p8=0.0357 loss_all=6.7515 init_gold_top10=0.1885 init_gold_top100=0.4529 rollout_applied_pos_frac=0.5625 init_acc_rollout_applied=0.1125 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.1083 logit_acc_rollout_kept=0.1037 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=1200 elapsed=37.5s lr=2.000000e-03 loss=6.4511 loss_recon=6.4511 loss_meanflow=0.0000 mean_model_t=0.2118 mean_corrupt_t=0.2118 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4957 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1155 corrupt_frac=1.0000 acc_corrupt=0.1155 loss_corrupt=6.4511 wrong_frac=0.7884 init_acc_corrupt=0.1212 acc_corrupt_t_0p0_0p2=0.0595 corrupt_frac_t_0p0_0p2=0.5476 acc_corrupt_t_0p2_0p4=0.1565 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.2815 corrupt_frac_t_0p4_0p6=0.0835 out_w_norm=4.3617 out_g_norm=0.8178 acc_corrupt_t_0p6_0p8=0.3852 corrupt_frac_t_0p6_0p8=0.0366 acc_corrupt_t_0p8_1p0=0.5366 corrupt_frac_t_0p8_1p0=0.0312 loss_all=6.2349 init_gold_top10=0.1833 init_gold_top100=0.4322 rollout_applied_pos_frac=0.4375 init_acc_rollout_applied=0.1633 init_acc_rollout_kept=0.0588 logit_acc_rollout_applied=0.1432 logit_acc_rollout_kept=0.0882 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=1600 elapsed=37.5s lr=2.000000e-03 loss=5.9906 loss_recon=5.9906 loss_meanflow=0.0000 mean_model_t=0.2091 mean_corrupt_t=0.2091 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5023 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1250 corrupt_frac=1.0000 acc_corrupt=0.1250 loss_corrupt=5.9906 wrong_frac=0.7911 init_acc_corrupt=0.1188 acc_corrupt_t_0p0_0p2=0.0629 corrupt_frac_t_0p0_0p2=0.5556 acc_corrupt_t_0p2_0p4=0.1717 corrupt_frac_t_0p2_0p4=0.3572 acc_corrupt_t_0p4_0p6=0.3178 corrupt_frac_t_0p4_0p6=0.0865 out_w_norm=5.4987 out_g_norm=0.3310 acc_corrupt_t_0p6_0p8=0.4333 corrupt_frac_t_0p6_0p8=0.0366 acc_corrupt_t_0p8_1p0=0.5137 corrupt_frac_t_0p8_1p0=0.0312 loss_all=5.7675 init_gold_top10=0.1940 init_gold_top100=0.4395 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.1511 init_acc_rollout_kept=0.0853 logit_acc_rollout_applied=0.1493 logit_acc_rollout_kept=0.1149 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=2000 elapsed=37.4s lr=2.000000e-03 loss=5.4943 loss_recon=5.4943 loss_meanflow=0.0000 mean_model_t=0.2104 mean_corrupt_t=0.2104 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5032 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1383 corrupt_frac=1.0000 acc_corrupt=0.1383 loss_corrupt=5.4943 wrong_frac=0.7895 init_acc_corrupt=0.1195 acc_corrupt_t_0p0_0p2=0.0679 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.1911 corrupt_frac_t_0p2_0p4=0.3658 acc_corrupt_t_0p4_0p6=0.3549 corrupt_frac_t_0p4_0p6=0.0809 out_w_norm=6.7494 out_g_norm=0.2532 acc_corrupt_t_0p6_0p8=0.4935 corrupt_frac_t_0p6_0p8=0.0353 acc_corrupt_t_0p8_1p0=0.6230 corrupt_frac_t_0p8_1p0=0.0312 loss_all=5.4116 init_gold_top10=0.1764 init_gold_top100=0.4898 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.0849 init_acc_rollout_kept=0.1106 logit_acc_rollout_applied=0.1191 logit_acc_rollout_kept=0.1371 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=2400 elapsed=37.4s lr=2.000000e-03 loss=4.8848 loss_recon=4.8848 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4970 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1534 corrupt_frac=1.0000 acc_corrupt=0.1534 loss_corrupt=4.8848 wrong_frac=0.7905 init_acc_corrupt=0.1196 acc_corrupt_t_0p0_0p2=0.0718 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.2144 corrupt_frac_t_0p2_0p4=0.3627 out_w_norm=8.0570 out_g_norm=0.2940 acc_corrupt_t_0p4_0p6=0.3997 corrupt_frac_t_0p4_0p6=0.0831 acc_corrupt_t_0p6_0p8=0.5749 corrupt_frac_t_0p6_0p8=0.0372 acc_corrupt_t_0p8_1p0=0.7373 corrupt_frac_t_0p8_1p0=0.0312 loss_all=4.4114 init_gold_top10=0.2211 init_gold_top100=0.6172 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.1576 init_acc_rollout_kept=0.0989 logit_acc_rollout_applied=0.2024 logit_acc_rollout_kept=0.1540 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=2800 elapsed=37.5s lr=2.000000e-03 loss=4.2577 loss_recon=4.2577 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4981 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1793 corrupt_frac=1.0000 acc_corrupt=0.1793 loss_corrupt=4.2577 wrong_frac=0.7899 init_acc_corrupt=0.1206 acc_corrupt_t_0p0_0p2=0.0785 corrupt_frac_t_0p0_0p2=0.5530 acc_corrupt_t_0p2_0p4=0.2532 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=0.4900 corrupt_frac_t_0p4_0p6=0.0863 acc_corrupt_t_0p6_0p8=0.6753 corrupt_frac_t_0p6_0p8=0.0361 out_w_norm=9.2749 out_g_norm=0.3343 acc_corrupt_t_0p8_1p0=0.8145 corrupt_frac_t_0p8_1p0=0.0312 loss_all=4.1313 init_gold_top10=0.2126 init_gold_top100=0.5775 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1626 init_acc_rollout_kept=0.1093 logit_acc_rollout_applied=0.2192 logit_acc_rollout_kept=0.1784 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=3200 elapsed=37.5s lr=2.000000e-03 loss=3.7527 loss_recon=3.7527 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5079 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2030 corrupt_frac=1.0000 acc_corrupt=0.2030 loss_corrupt=3.7527 wrong_frac=0.7903 init_acc_corrupt=0.1207 acc_corrupt_t_0p0_0p2=0.0879 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.2965 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=0.5306 corrupt_frac_t_0p4_0p6=0.0850 out_w_norm=10.1655 out_g_norm=0.3946 acc_corrupt_t_0p6_0p8=0.7050 corrupt_frac_t_0p6_0p8=0.0338 acc_corrupt_t_0p8_1p0=0.8413 corrupt_frac_t_0p8_1p0=0.0312 loss_all=3.6608 init_gold_top10=0.2162 init_gold_top100=0.6223 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.1540 init_acc_rollout_kept=0.0570 logit_acc_rollout_applied=0.2421 logit_acc_rollout_kept=0.1555 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=3600 elapsed=37.4s lr=2.000000e-03 loss=3.3035 loss_recon=3.3035 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4956 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2399 corrupt_frac=1.0000 acc_corrupt=0.2399 loss_corrupt=3.3035 wrong_frac=0.7892 init_acc_corrupt=0.1235 acc_corrupt_t_0p0_0p2=0.1037 corrupt_frac_t_0p0_0p2=0.5547 acc_corrupt_t_0p2_0p4=0.3581 corrupt_frac_t_0p2_0p4=0.3554 acc_corrupt_t_0p4_0p6=0.5989 corrupt_frac_t_0p4_0p6=0.0877 out_w_norm=10.7195 out_g_norm=0.5110 acc_corrupt_t_0p6_0p8=0.7433 corrupt_frac_t_0p6_0p8=0.0358 acc_corrupt_t_0p8_1p0=0.8540 corrupt_frac_t_0p8_1p0=0.0312 loss_all=3.0285 init_gold_top10=0.2676 init_gold_top100=0.6202 rollout_applied_pos_frac=0.4375 init_acc_rollout_applied=0.1009 init_acc_rollout_kept=0.1176 logit_acc_rollout_applied=0.2540 logit_acc_rollout_kept=0.2764 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=4000 elapsed=37.5s lr=2.000000e-03 loss=2.8950 loss_recon=2.8950 loss_meanflow=0.0000 mean_model_t=0.2090 mean_corrupt_t=0.2090 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2938 corrupt_frac=1.0000 acc_corrupt=0.2938 loss_corrupt=2.8950 wrong_frac=0.7911 init_acc_corrupt=0.1224 acc_corrupt_t_0p0_0p2=0.1292 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=0.4540 corrupt_frac_t_0p2_0p4=0.3541 acc_corrupt_t_0p4_0p6=0.6939 corrupt_frac_t_0p4_0p6=0.0843 out_w_norm=11.0862 out_g_norm=0.6754 acc_corrupt_t_0p6_0p8=0.8143 corrupt_frac_t_0p6_0p8=0.0341 acc_corrupt_t_0p8_1p0=0.9004 corrupt_frac_t_0p8_1p0=0.0312 loss_all=3.0296 init_gold_top10=0.2572 init_gold_top100=0.5604 rollout_applied_pos_frac=0.3750 init_acc_rollout_applied=0.0764 init_acc_rollout_kept=0.1180 logit_acc_rollout_applied=0.2254 logit_acc_rollout_kept=0.3271 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024", + "batch_size": 32, + "grad_accum": 4, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1840ctx1024/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=4400 elapsed=38.3s lr=2.000000e-03 loss=2.4945 loss_recon=2.4945 loss_meanflow=0.0000 mean_model_t=0.2092 mean_corrupt_t=0.2092 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5055 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3683 corrupt_frac=1.0000 acc_corrupt=0.3683 loss_corrupt=2.4945 wrong_frac=0.7908 init_acc_corrupt=0.1240 acc_corrupt_t_0p0_0p2=0.1675 corrupt_frac_t_0p0_0p2=0.5551 acc_corrupt_t_0p2_0p4=0.5746 corrupt_frac_t_0p2_0p4=0.3604 acc_corrupt_t_0p4_0p6=0.7983 corrupt_frac_t_0p4_0p6=0.0822 acc_corrupt_t_0p6_0p8=0.8784 corrupt_frac_t_0p6_0p8=0.0356 out_w_norm=11.3758 out_g_norm=0.7552 loss_all=2.5911 init_gold_top10=0.3336 init_gold_top100=0.6645 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.0988 init_acc_rollout_kept=0.0578 logit_acc_rollout_applied=0.3668 logit_acc_rollout_kept=0.3079 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=4800 elapsed=37.6s lr=2.000000e-03 loss=2.1611 loss_recon=2.1611 loss_meanflow=0.0000 mean_model_t=0.2087 mean_corrupt_t=0.2087 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5035 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4426 corrupt_frac=1.0000 acc_corrupt=0.4426 loss_corrupt=2.1611 wrong_frac=0.7913 init_acc_corrupt=0.1255 acc_corrupt_t_0p0_0p2=0.2089 corrupt_frac_t_0p0_0p2=0.5574 acc_corrupt_t_0p2_0p4=0.7016 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.8862 corrupt_frac_t_0p4_0p6=0.0807 acc_corrupt_t_0p8_1p0=0.9312 corrupt_frac_t_0p8_1p0=0.0312 out_w_norm=11.6450 out_g_norm=0.8699 acc_corrupt_t_0p6_0p8=0.9271 corrupt_frac_t_0p6_0p8=0.0357 loss_all=2.2175 init_gold_top10=0.4112 init_gold_top100=0.6921 rollout_applied_pos_frac=0.5625 init_acc_rollout_applied=0.1286 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.4788 logit_acc_rollout_kept=0.4031 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=5200 elapsed=37.7s lr=2.000000e-03 loss=1.8630 loss_recon=1.8630 loss_meanflow=0.0000 mean_model_t=0.2118 mean_corrupt_t=0.2118 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4957 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5171 corrupt_frac=1.0000 acc_corrupt=0.5171 loss_corrupt=1.8630 wrong_frac=0.7884 init_acc_corrupt=0.1321 acc_corrupt_t_0p0_0p2=0.2547 corrupt_frac_t_0p0_0p2=0.5476 acc_corrupt_t_0p2_0p4=0.8082 corrupt_frac_t_0p2_0p4=0.3645 acc_corrupt_t_0p4_0p6=0.9420 corrupt_frac_t_0p4_0p6=0.0835 out_w_norm=11.8612 out_g_norm=0.9278 acc_corrupt_t_0p6_0p8=0.9599 corrupt_frac_t_0p6_0p8=0.0366 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.0312 loss_all=1.9265 init_gold_top10=0.4385 init_gold_top100=0.6085 rollout_applied_pos_frac=0.4375 init_acc_rollout_applied=0.1980 init_acc_rollout_kept=0.0588 logit_acc_rollout_applied=0.7255 logit_acc_rollout_kept=0.3106 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=5600 elapsed=37.6s lr=2.000000e-03 loss=1.6395 loss_recon=1.6395 loss_meanflow=0.0000 mean_model_t=0.2091 mean_corrupt_t=0.2091 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5023 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5740 corrupt_frac=1.0000 acc_corrupt=0.5740 loss_corrupt=1.6395 wrong_frac=0.7911 init_acc_corrupt=0.1323 acc_corrupt_t_0p0_0p2=0.3145 corrupt_frac_t_0p0_0p2=0.5556 acc_corrupt_t_0p2_0p4=0.8804 corrupt_frac_t_0p2_0p4=0.3572 acc_corrupt_t_0p4_0p6=0.9724 corrupt_frac_t_0p4_0p6=0.0865 out_w_norm=12.0236 out_g_norm=0.9774 acc_corrupt_t_0p6_0p8=0.9795 corrupt_frac_t_0p6_0p8=0.0366 acc_corrupt_t_0p8_1p0=0.9873 corrupt_frac_t_0p8_1p0=0.0312 loss_all=1.7346 init_gold_top10=0.3990 init_gold_top100=0.5863 rollout_applied_pos_frac=0.4062 init_acc_rollout_applied=0.1877 init_acc_rollout_kept=0.0853 logit_acc_rollout_applied=0.6870 logit_acc_rollout_kept=0.4878 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=6000 elapsed=37.6s lr=2.000000e-03 loss=1.3980 loss_recon=1.3980 loss_meanflow=0.0000 mean_model_t=0.2104 mean_corrupt_t=0.2104 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5032 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6327 corrupt_frac=1.0000 acc_corrupt=0.6327 loss_corrupt=1.3980 wrong_frac=0.7895 init_acc_corrupt=0.1356 acc_corrupt_t_0p0_0p2=0.3876 corrupt_frac_t_0p0_0p2=0.5505 acc_corrupt_t_0p2_0p4=0.9209 corrupt_frac_t_0p2_0p4=0.3658 acc_corrupt_t_0p4_0p6=0.9857 corrupt_frac_t_0p4_0p6=0.0809 out_w_norm=12.1409 out_g_norm=0.9930 acc_corrupt_t_0p6_0p8=0.9875 corrupt_frac_t_0p6_0p8=0.0353 acc_corrupt_t_0p8_1p0=0.9740 corrupt_frac_t_0p8_1p0=0.0312 loss_all=2.0586 init_gold_top10=0.4348 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.1116 init_acc_rollout_kept=0.1106 logit_acc_rollout_applied=0.5888 logit_acc_rollout_kept=0.4600 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=6400 elapsed=37.5s lr=2.000000e-03 loss=1.2548 loss_recon=1.2548 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4970 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6669 corrupt_frac=1.0000 acc_corrupt=0.6669 loss_corrupt=1.2548 wrong_frac=0.7905 init_acc_corrupt=0.1381 acc_corrupt_t_0p0_0p2=0.4306 corrupt_frac_t_0p0_0p2=0.5511 acc_corrupt_t_0p2_0p4=0.9487 corrupt_frac_t_0p2_0p4=0.3627 out_w_norm=12.2261 out_g_norm=0.9112 acc_corrupt_t_0p4_0p6=0.9921 corrupt_frac_t_0p4_0p6=0.0831 acc_corrupt_t_0p6_0p8=0.9925 corrupt_frac_t_0p6_0p8=0.0372 acc_corrupt_t_0p8_1p0=0.9829 corrupt_frac_t_0p8_1p0=0.0312 loss_all=1.0855 init_gold_top10=0.5554 init_gold_top100=0.6855 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.2027 init_acc_rollout_kept=0.0989 logit_acc_rollout_applied=0.8286 logit_acc_rollout_kept=0.5480 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=6800 elapsed=37.3s lr=2.000000e-03 loss=1.1174 loss_recon=1.1174 loss_meanflow=0.0000 mean_model_t=0.2102 mean_corrupt_t=0.2102 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4981 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6984 corrupt_frac=1.0000 acc_corrupt=0.6984 loss_corrupt=1.1174 wrong_frac=0.7899 init_acc_corrupt=0.1409 acc_corrupt_t_0p0_0p2=0.4781 corrupt_frac_t_0p0_0p2=0.5530 acc_corrupt_t_0p2_0p4=0.9650 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=0.9951 corrupt_frac_t_0p4_0p6=0.0863 acc_corrupt_t_0p6_0p8=0.9940 corrupt_frac_t_0p6_0p8=0.0361 out_w_norm=12.2909 out_g_norm=0.8889 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0312 loss_all=1.3472 init_gold_top10=0.4938 init_gold_top100=0.6382 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.2066 init_acc_rollout_kept=0.1093 logit_acc_rollout_applied=0.7921 logit_acc_rollout_kept=0.5591 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=7200 elapsed=37.6s lr=2.000000e-03 loss=1.0126 loss_recon=1.0126 loss_meanflow=0.0000 mean_model_t=0.2096 mean_corrupt_t=0.2096 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5079 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7211 corrupt_frac=1.0000 acc_corrupt=0.7211 loss_corrupt=1.0126 wrong_frac=0.7903 init_acc_corrupt=0.1416 acc_corrupt_t_0p0_0p2=0.5124 corrupt_frac_t_0p0_0p2=0.5517 acc_corrupt_t_0p2_0p4=0.9736 corrupt_frac_t_0p2_0p4=0.3616 acc_corrupt_t_0p4_0p6=0.9970 corrupt_frac_t_0p4_0p6=0.0850 out_w_norm=12.3265 out_g_norm=0.8711 acc_corrupt_t_0p6_0p8=0.9957 corrupt_frac_t_0p6_0p8=0.0338 acc_corrupt_t_0p8_1p0=0.9839 corrupt_frac_t_0p8_1p0=0.0312 loss_all=1.1469 init_gold_top10=0.5062 init_gold_top100=0.6721 rollout_applied_pos_frac=0.5312 init_acc_rollout_applied=0.2005 init_acc_rollout_kept=0.0570 logit_acc_rollout_applied=0.7336 logit_acc_rollout_kept=0.5972 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128.log new file mode 100644 index 0000000000000000000000000000000000000000..bb7cbb8e0ade2f972fc3ee75174b0290c76188c1 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128.log @@ -0,0 +1,4000 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=35.0s lr=2.000000e-03 loss=7.7220 loss_recon=7.7220 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0950 corrupt_frac=1.0000 acc_corrupt=0.0950 loss_corrupt=7.7220 wrong_frac=0.7918 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.0500 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1258 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2500 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3501 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0069 out_g_norm=1.0877 loss_all=7.4664 init_gold_top10=0.2041 init_gold_top100=0.4163 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1140 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.1042 logit_acc_rollout_kept=0.0971 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=34.4s lr=2.000000e-03 loss=7.0819 loss_recon=7.0819 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1044 corrupt_frac=1.0000 acc_corrupt=0.1044 loss_corrupt=7.0819 wrong_frac=0.7893 init_acc_corrupt=0.1181 acc_corrupt_t_0p0_0p2=0.0562 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1398 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2543 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.3509 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=2.8670 out_g_norm=1.7742 acc_corrupt_t_0p8_1p0=0.4303 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.7595 init_gold_top10=0.1930 init_gold_top100=0.4428 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.0958 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.0999 logit_acc_rollout_kept=0.1074 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=34.4s lr=2.000000e-03 loss=6.4702 loss_recon=6.4702 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1121 corrupt_frac=1.0000 acc_corrupt=0.1121 loss_corrupt=6.4702 wrong_frac=0.7934 init_acc_corrupt=0.1146 acc_corrupt_t_0p0_0p2=0.0590 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1548 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.2821 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.3973 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=4.3630 out_g_norm=1.3336 acc_corrupt_t_0p8_1p0=0.5381 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.2092 init_gold_top10=0.1972 init_gold_top100=0.4450 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1294 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.1225 logit_acc_rollout_kept=0.1171 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=34.4s lr=2.000000e-03 loss=5.9923 loss_recon=5.9923 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1245 corrupt_frac=1.0000 acc_corrupt=0.1245 loss_corrupt=5.9923 wrong_frac=0.7914 init_acc_corrupt=0.1168 acc_corrupt_t_0p0_0p2=0.0634 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.1719 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.3160 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=5.4942 out_g_norm=0.5159 acc_corrupt_t_0p6_0p8=0.4370 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.5767 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8497 init_gold_top10=0.1915 init_gold_top100=0.4411 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.0906 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.1107 logit_acc_rollout_kept=0.1270 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=34.4s lr=2.000000e-03 loss=5.5183 loss_recon=5.5183 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1353 corrupt_frac=1.0000 acc_corrupt=0.1353 loss_corrupt=5.5183 wrong_frac=0.7930 init_acc_corrupt=0.1156 acc_corrupt_t_0p0_0p2=0.0675 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.1894 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.3541 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.4920 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.5605 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=6.7472 out_g_norm=0.4287 loss_all=5.1460 init_gold_top10=0.2073 init_gold_top100=0.5265 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1125 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.1426 logit_acc_rollout_kept=0.1574 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=34.4s lr=2.000000e-03 loss=4.9011 loss_recon=4.9011 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1517 corrupt_frac=1.0000 acc_corrupt=0.1517 loss_corrupt=4.9011 wrong_frac=0.7925 init_acc_corrupt=0.1163 acc_corrupt_t_0p0_0p2=0.0728 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.2151 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.4015 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.5663 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=8.0534 out_g_norm=0.3984 acc_corrupt_t_0p8_1p0=0.6445 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7086 init_gold_top10=0.1904 init_gold_top100=0.5562 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1038 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.1464 logit_acc_rollout_kept=0.1534 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=34.4s lr=2.000000e-03 loss=4.2783 loss_recon=4.2783 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1782 corrupt_frac=1.0000 acc_corrupt=0.1782 loss_corrupt=4.2783 wrong_frac=0.7907 init_acc_corrupt=0.1198 acc_corrupt_t_0p0_0p2=0.0780 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.2545 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.4886 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.6649 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=9.2741 out_g_norm=0.4910 acc_corrupt_t_0p8_1p0=0.8584 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.9263 init_gold_top10=0.2201 init_gold_top100=0.6252 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1293 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.2018 logit_acc_rollout_kept=0.1978 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=34.4s lr=2.000000e-03 loss=3.7831 loss_recon=3.7831 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2019 corrupt_frac=1.0000 acc_corrupt=0.2019 loss_corrupt=3.7831 wrong_frac=0.7901 init_acc_corrupt=0.1211 acc_corrupt_t_0p0_0p2=0.0867 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.2965 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.5335 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=10.1643 out_g_norm=0.6139 acc_corrupt_t_0p6_0p8=0.6948 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.8398 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.3816 init_gold_top10=0.2517 init_gold_top100=0.6582 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.1512 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.2442 logit_acc_rollout_kept=0.2381 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=34.4s lr=2.000000e-03 loss=3.3383 loss_recon=3.3383 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2376 corrupt_frac=1.0000 acc_corrupt=0.2376 loss_corrupt=3.3383 wrong_frac=0.7895 init_acc_corrupt=0.1232 acc_corrupt_t_0p0_0p2=0.1010 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.3552 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.5990 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.7443 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=10.7223 out_g_norm=0.7962 acc_corrupt_t_0p8_1p0=0.8477 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.1953 init_gold_top10=0.2654 init_gold_top100=0.6566 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1222 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.2537 logit_acc_rollout_kept=0.2421 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=34.4s lr=2.000000e-03 loss=2.9156 loss_recon=2.9156 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2931 corrupt_frac=1.0000 acc_corrupt=0.2931 loss_corrupt=2.9156 wrong_frac=0.7888 init_acc_corrupt=0.1252 acc_corrupt_t_0p0_0p2=0.1255 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.4465 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.6915 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=11.1076 out_g_norm=0.9560 acc_corrupt_t_0p6_0p8=0.8070 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9100 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.7019 init_gold_top10=0.3382 init_gold_top100=0.6432 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.1677 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.3772 logit_acc_rollout_kept=0.2949 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=35.0s lr=2.000000e-03 loss=2.5376 loss_recon=2.5376 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3609 corrupt_frac=1.0000 acc_corrupt=0.3609 loss_corrupt=2.5376 wrong_frac=0.7918 init_acc_corrupt=0.1230 acc_corrupt_t_0p0_0p2=0.1609 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.5659 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.7946 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.8695 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.4167 out_g_norm=1.1839 loss_all=2.1696 init_gold_top10=0.3945 init_gold_top100=0.6425 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1351 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.4577 logit_acc_rollout_kept=0.3925 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=34.4s lr=2.000000e-03 loss=2.1489 loss_recon=2.1489 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4449 corrupt_frac=1.0000 acc_corrupt=0.4449 loss_corrupt=2.1489 wrong_frac=0.7893 init_acc_corrupt=0.1272 acc_corrupt_t_0p0_0p2=0.2136 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.6928 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.8790 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9269 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=11.6856 out_g_norm=1.3283 acc_corrupt_t_0p8_1p0=0.9535 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.9722 init_gold_top10=0.4314 init_gold_top100=0.6743 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1163 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.5031 logit_acc_rollout_kept=0.4724 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=34.4s lr=2.000000e-03 loss=1.9016 loss_recon=1.9016 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5066 corrupt_frac=1.0000 acc_corrupt=0.5066 loss_corrupt=1.9016 wrong_frac=0.7934 init_acc_corrupt=0.1251 acc_corrupt_t_0p0_0p2=0.2592 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.8003 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9406 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9602 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=11.9148 out_g_norm=1.5043 acc_corrupt_t_0p8_1p0=0.9733 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.0515 init_gold_top10=0.4219 init_gold_top100=0.6368 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1533 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.6019 logit_acc_rollout_kept=0.4453 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=34.4s lr=2.000000e-03 loss=1.6190 loss_recon=1.6190 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5743 corrupt_frac=1.0000 acc_corrupt=0.5743 loss_corrupt=1.6190 wrong_frac=0.7914 init_acc_corrupt=0.1305 acc_corrupt_t_0p0_0p2=0.3188 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.8776 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9705 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.0878 out_g_norm=1.5617 acc_corrupt_t_0p6_0p8=0.9779 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.5940 init_gold_top10=0.4113 init_gold_top100=0.6148 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1169 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.6099 logit_acc_rollout_kept=0.5352 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=34.4s lr=2.000000e-03 loss=1.4132 loss_recon=1.4132 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6248 corrupt_frac=1.0000 acc_corrupt=0.6248 loss_corrupt=1.4132 wrong_frac=0.7930 init_acc_corrupt=0.1317 acc_corrupt_t_0p0_0p2=0.3856 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9212 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9851 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9860 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9639 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.1889 out_g_norm=1.4521 loss_all=1.3110 init_gold_top10=0.5051 init_gold_top100=0.6796 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1485 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.7310 logit_acc_rollout_kept=0.5571 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=34.4s lr=2.000000e-03 loss=1.2086 loss_recon=1.2086 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6721 corrupt_frac=1.0000 acc_corrupt=0.6721 loss_corrupt=1.2086 wrong_frac=0.7925 init_acc_corrupt=0.1344 acc_corrupt_t_0p0_0p2=0.4498 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9488 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9925 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9918 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.2542 out_g_norm=1.3169 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.3674 init_gold_top10=0.4757 init_gold_top100=0.6593 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1388 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.6783 logit_acc_rollout_kept=0.5529 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=34.4s lr=2.000000e-03 loss=1.0481 loss_recon=1.0481 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7091 corrupt_frac=1.0000 acc_corrupt=0.7091 loss_corrupt=1.0481 wrong_frac=0.7907 init_acc_corrupt=0.1398 acc_corrupt_t_0p0_0p2=0.4991 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9651 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9959 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9948 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.3068 out_g_norm=1.2812 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.9482 init_gold_top10=0.5454 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1737 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.8505 logit_acc_rollout_kept=0.6395 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=34.4s lr=2.000000e-03 loss=0.9610 loss_recon=0.9610 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7300 corrupt_frac=1.0000 acc_corrupt=0.7300 loss_corrupt=0.9610 wrong_frac=0.7901 init_acc_corrupt=0.1414 acc_corrupt_t_0p0_0p2=0.5292 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9761 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9976 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.3512 out_g_norm=1.2300 acc_corrupt_t_0p6_0p8=0.9953 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8295 init_gold_top10=0.5786 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.1951 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.8199 logit_acc_rollout_kept=0.6901 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=34.4s lr=2.000000e-03 loss=0.8589 loss_recon=0.8589 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7542 corrupt_frac=1.0000 acc_corrupt=0.7542 loss_corrupt=0.8589 wrong_frac=0.7895 init_acc_corrupt=0.1442 acc_corrupt_t_0p0_0p2=0.5657 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9838 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9982 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9968 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.3626 out_g_norm=1.1791 acc_corrupt_t_0p8_1p0=0.9883 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8784 init_gold_top10=0.5510 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1672 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.7921 logit_acc_rollout_kept=0.6906 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=34.4s lr=2.000000e-03 loss=0.7667 loss_recon=0.7667 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7759 corrupt_frac=1.0000 acc_corrupt=0.7759 loss_corrupt=0.7667 wrong_frac=0.7888 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.6009 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9893 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9988 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.3708 out_g_norm=0.9733 acc_corrupt_t_0p6_0p8=0.9977 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9898 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7178 init_gold_top10=0.5525 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2093 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.8699 logit_acc_rollout_kept=0.7213 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=35.0s lr=2.000000e-03 loss=0.6877 loss_recon=0.6877 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7945 corrupt_frac=1.0000 acc_corrupt=0.7945 loss_corrupt=0.6877 wrong_frac=0.7918 init_acc_corrupt=0.1438 acc_corrupt_t_0p0_0p2=0.6356 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9922 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9990 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9971 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.3718 out_g_norm=1.0307 loss_all=0.4577 init_gold_top10=0.5648 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1830 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9212 logit_acc_rollout_kept=0.7980 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=34.5s lr=2.000000e-03 loss=0.6092 loss_recon=0.6092 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8156 corrupt_frac=1.0000 acc_corrupt=0.8156 loss_corrupt=0.6092 wrong_frac=0.7893 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.6693 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9948 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9991 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9977 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.3592 out_g_norm=0.9482 acc_corrupt_t_0p8_1p0=0.9925 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6124 init_gold_top10=0.5840 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1525 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.8268 logit_acc_rollout_kept=0.7892 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=34.4s lr=2.000000e-03 loss=0.5764 loss_recon=0.5764 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8225 corrupt_frac=1.0000 acc_corrupt=0.8225 loss_corrupt=0.5764 wrong_frac=0.7934 init_acc_corrupt=0.1424 acc_corrupt_t_0p0_0p2=0.6877 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9967 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9993 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9980 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.3385 out_g_norm=0.9341 acc_corrupt_t_0p8_1p0=0.9938 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.7582 init_gold_top10=0.5227 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1833 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.8488 logit_acc_rollout_kept=0.7415 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=34.4s lr=2.000000e-03 loss=0.5119 loss_recon=0.5119 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8407 corrupt_frac=1.0000 acc_corrupt=0.8407 loss_corrupt=0.5119 wrong_frac=0.7914 init_acc_corrupt=0.1456 acc_corrupt_t_0p0_0p2=0.7153 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9992 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.3127 out_g_norm=0.8978 acc_corrupt_t_0p6_0p8=0.9984 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9976 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.5930 init_gold_top10=0.4953 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1424 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.7721 logit_acc_rollout_kept=0.8465 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=34.4s lr=2.000000e-03 loss=0.4811 loss_recon=0.4811 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8486 corrupt_frac=1.0000 acc_corrupt=0.8486 loss_corrupt=0.4811 wrong_frac=0.7930 init_acc_corrupt=0.1438 acc_corrupt_t_0p0_0p2=0.7323 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9984 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9983 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9951 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.2797 out_g_norm=0.8356 loss_all=0.4408 init_gold_top10=0.5882 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1713 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.8782 logit_acc_rollout_kept=0.8320 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=34.4s lr=2.000000e-03 loss=0.4533 loss_recon=0.4533 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8560 corrupt_frac=1.0000 acc_corrupt=0.8560 loss_corrupt=0.4533 wrong_frac=0.7925 init_acc_corrupt=0.1441 acc_corrupt_t_0p0_0p2=0.7443 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9987 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.2422 out_g_norm=0.7357 acc_corrupt_t_0p8_1p0=0.9922 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.6222 init_gold_top10=0.5561 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1518 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8363 logit_acc_rollout_kept=0.8000 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=34.4s lr=2.000000e-03 loss=0.4389 loss_recon=0.4389 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8610 corrupt_frac=1.0000 acc_corrupt=0.8610 loss_corrupt=0.4389 wrong_frac=0.7907 init_acc_corrupt=0.1472 acc_corrupt_t_0p0_0p2=0.7502 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9995 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9989 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.2147 out_g_norm=0.7702 acc_corrupt_t_0p8_1p0=0.9785 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3821 init_gold_top10=0.5825 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1862 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9114 logit_acc_rollout_kept=0.8281 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=34.4s lr=2.000000e-03 loss=0.4172 loss_recon=0.4172 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8662 corrupt_frac=1.0000 acc_corrupt=0.8662 loss_corrupt=0.4172 wrong_frac=0.7901 init_acc_corrupt=0.1475 acc_corrupt_t_0p0_0p2=0.7594 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.1980 out_g_norm=0.6951 acc_corrupt_t_0p6_0p8=0.9961 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9844 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4915 init_gold_top10=0.6036 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2070 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.8623 logit_acc_rollout_kept=0.8615 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=34.4s lr=2.000000e-03 loss=0.3944 loss_recon=0.3944 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8726 corrupt_frac=1.0000 acc_corrupt=0.8726 loss_corrupt=0.3944 wrong_frac=0.7895 init_acc_corrupt=0.1495 acc_corrupt_t_0p0_0p2=0.7695 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9990 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.1955 out_g_norm=0.5870 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4065 init_gold_top10=0.5993 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1793 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.8837 logit_acc_rollout_kept=0.8361 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=34.5s lr=2.000000e-03 loss=0.3734 loss_recon=0.3734 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8792 corrupt_frac=1.0000 acc_corrupt=0.8792 loss_corrupt=0.3734 wrong_frac=0.7888 init_acc_corrupt=0.1506 acc_corrupt_t_0p0_0p2=0.7812 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.1907 out_g_norm=0.5829 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9959 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3789 init_gold_top10=0.5671 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2164 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9171 logit_acc_rollout_kept=0.8673 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=35.0s lr=2.000000e-03 loss=0.3631 loss_recon=0.3631 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8824 corrupt_frac=1.0000 acc_corrupt=0.8824 loss_corrupt=0.3631 wrong_frac=0.7918 init_acc_corrupt=0.1475 acc_corrupt_t_0p0_0p2=0.7888 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9992 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.1944 out_g_norm=0.5417 loss_all=0.1875 init_gold_top10=0.5797 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1917 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9490 logit_acc_rollout_kept=0.9243 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=34.4s lr=2.000000e-03 loss=0.3491 loss_recon=0.3491 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8885 corrupt_frac=1.0000 acc_corrupt=0.8885 loss_corrupt=0.3491 wrong_frac=0.7893 init_acc_corrupt=0.1495 acc_corrupt_t_0p0_0p2=0.7982 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9988 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.2090 out_g_norm=0.5324 acc_corrupt_t_0p8_1p0=0.9948 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2984 init_gold_top10=0.6017 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9227 logit_acc_rollout_kept=0.8829 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=34.4s lr=2.000000e-03 loss=0.3566 loss_recon=0.3566 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8855 corrupt_frac=1.0000 acc_corrupt=0.8855 loss_corrupt=0.3566 wrong_frac=0.7934 init_acc_corrupt=0.1451 acc_corrupt_t_0p0_0p2=0.7973 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.2149 out_g_norm=0.5043 acc_corrupt_t_0p8_1p0=0.9977 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4997 init_gold_top10=0.5437 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1882 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9029 logit_acc_rollout_kept=0.8060 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=34.4s lr=2.000000e-03 loss=0.3312 loss_recon=0.3312 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8933 corrupt_frac=1.0000 acc_corrupt=0.8933 loss_corrupt=0.3312 wrong_frac=0.7914 init_acc_corrupt=0.1480 acc_corrupt_t_0p0_0p2=0.8083 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.2369 out_g_norm=0.4880 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9985 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4743 init_gold_top10=0.5112 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1455 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.7949 logit_acc_rollout_kept=0.8951 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=34.4s lr=2.000000e-03 loss=0.3237 loss_recon=0.3237 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8952 corrupt_frac=1.0000 acc_corrupt=0.8952 loss_corrupt=0.3237 wrong_frac=0.7930 init_acc_corrupt=0.1459 acc_corrupt_t_0p0_0p2=0.8142 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.2522 out_g_norm=0.5223 loss_all=0.2948 init_gold_top10=0.5969 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1759 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9261 logit_acc_rollout_kept=0.8813 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=34.4s lr=2.000000e-03 loss=0.3085 loss_recon=0.3085 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9002 corrupt_frac=1.0000 acc_corrupt=0.9002 loss_corrupt=0.3085 wrong_frac=0.7925 init_acc_corrupt=0.1458 acc_corrupt_t_0p0_0p2=0.8225 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9991 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.2732 out_g_norm=0.4286 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3445 init_gold_top10=0.5672 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1570 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8669 logit_acc_rollout_kept=0.8984 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=34.4s lr=2.000000e-03 loss=0.2994 loss_recon=0.2994 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9040 corrupt_frac=1.0000 acc_corrupt=0.9040 loss_corrupt=0.2994 wrong_frac=0.7907 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.8272 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.2962 out_g_norm=0.3777 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2271 init_gold_top10=0.5896 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1907 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9546 logit_acc_rollout_kept=0.8891 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=34.5s lr=2.000000e-03 loss=0.2881 loss_recon=0.2881 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9070 corrupt_frac=1.0000 acc_corrupt=0.9070 loss_corrupt=0.2881 wrong_frac=0.7901 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.8325 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.3216 out_g_norm=0.3791 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3038 init_gold_top10=0.6248 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2090 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9165 logit_acc_rollout_kept=0.8792 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=34.6s lr=2.000000e-03 loss=0.2988 loss_recon=0.2988 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9048 corrupt_frac=1.0000 acc_corrupt=0.9048 loss_corrupt=0.2988 wrong_frac=0.7895 init_acc_corrupt=0.1507 acc_corrupt_t_0p0_0p2=0.8279 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9980 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.3426 out_g_norm=0.3916 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3584 init_gold_top10=0.6037 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1817 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.8907 logit_acc_rollout_kept=0.8824 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=34.6s lr=2.000000e-03 loss=0.2800 loss_recon=0.2800 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9100 corrupt_frac=1.0000 acc_corrupt=0.9100 loss_corrupt=0.2800 wrong_frac=0.7888 init_acc_corrupt=0.1516 acc_corrupt_t_0p0_0p2=0.8370 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.3688 out_g_norm=0.3642 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2645 init_gold_top10=0.5727 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2187 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9381 logit_acc_rollout_kept=0.9141 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=35.1s lr=2.000000e-03 loss=0.2760 loss_recon=0.2760 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9124 corrupt_frac=1.0000 acc_corrupt=0.9124 loss_corrupt=0.2760 wrong_frac=0.7918 init_acc_corrupt=0.1485 acc_corrupt_t_0p0_0p2=0.8424 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.3989 out_g_norm=0.3045 loss_all=0.1085 init_gold_top10=0.5878 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1936 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9787 logit_acc_rollout_kept=0.9591 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=34.4s lr=2.000000e-03 loss=0.2607 loss_recon=0.2607 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9173 corrupt_frac=1.0000 acc_corrupt=0.9173 loss_corrupt=0.2607 wrong_frac=0.7893 init_acc_corrupt=0.1504 acc_corrupt_t_0p0_0p2=0.8502 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.4185 out_g_norm=0.2946 acc_corrupt_t_0p8_1p0=0.9928 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2512 init_gold_top10=0.6050 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1604 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9263 logit_acc_rollout_kept=0.9140 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=34.4s lr=2.000000e-03 loss=0.2610 loss_recon=0.2610 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9155 corrupt_frac=1.0000 acc_corrupt=0.9155 loss_corrupt=0.2610 wrong_frac=0.7934 init_acc_corrupt=0.1458 acc_corrupt_t_0p0_0p2=0.8503 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.4486 out_g_norm=0.2763 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3202 init_gold_top10=0.5526 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1899 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9123 logit_acc_rollout_kept=0.8795 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=34.4s lr=2.000000e-03 loss=0.2668 loss_recon=0.2668 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9148 corrupt_frac=1.0000 acc_corrupt=0.9148 loss_corrupt=0.2668 wrong_frac=0.7914 init_acc_corrupt=0.1486 acc_corrupt_t_0p0_0p2=0.8470 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.4733 out_g_norm=0.2742 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.4062 init_gold_top10=0.5128 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1473 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.7988 logit_acc_rollout_kept=0.9221 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=34.4s lr=2.000000e-03 loss=0.2623 loss_recon=0.2623 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9155 corrupt_frac=1.0000 acc_corrupt=0.9155 loss_corrupt=0.2623 wrong_frac=0.7930 init_acc_corrupt=0.1465 acc_corrupt_t_0p0_0p2=0.8501 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9971 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.4881 out_g_norm=0.2542 loss_all=0.1997 init_gold_top10=0.6073 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1770 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9311 logit_acc_rollout_kept=0.9287 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=34.4s lr=2.000000e-03 loss=0.2436 loss_recon=0.2436 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9217 corrupt_frac=1.0000 acc_corrupt=0.9217 loss_corrupt=0.2436 wrong_frac=0.7925 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.8607 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.5086 out_g_norm=0.2355 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3165 init_gold_top10=0.5639 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1582 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8592 logit_acc_rollout_kept=0.9255 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=34.4s lr=2.000000e-03 loss=0.2556 loss_recon=0.2556 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9199 corrupt_frac=1.0000 acc_corrupt=0.9199 loss_corrupt=0.2556 wrong_frac=0.7907 init_acc_corrupt=0.1494 acc_corrupt_t_0p0_0p2=0.8558 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.5236 out_g_norm=0.2470 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2806 init_gold_top10=0.5875 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1908 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9533 logit_acc_rollout_kept=0.8846 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=34.4s lr=2.000000e-03 loss=0.2474 loss_recon=0.2474 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9214 corrupt_frac=1.0000 acc_corrupt=0.9214 loss_corrupt=0.2474 wrong_frac=0.7901 init_acc_corrupt=0.1493 acc_corrupt_t_0p0_0p2=0.8584 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.5392 out_g_norm=0.2405 acc_corrupt_t_0p6_0p8=0.9992 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9932 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2540 init_gold_top10=0.6247 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2100 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9474 logit_acc_rollout_kept=0.8970 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=34.4s lr=2.000000e-03 loss=0.2396 loss_recon=0.2396 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9235 corrupt_frac=1.0000 acc_corrupt=0.9235 loss_corrupt=0.2396 wrong_frac=0.7895 init_acc_corrupt=0.1511 acc_corrupt_t_0p0_0p2=0.8616 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.5609 out_g_norm=0.2499 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3054 init_gold_top10=0.5989 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1824 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9102 logit_acc_rollout_kept=0.8998 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=34.4s lr=2.000000e-03 loss=0.2470 loss_recon=0.2470 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9222 corrupt_frac=1.0000 acc_corrupt=0.9222 loss_corrupt=0.2470 wrong_frac=0.7888 init_acc_corrupt=0.1518 acc_corrupt_t_0p0_0p2=0.8590 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.5833 out_g_norm=0.2204 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9984 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2933 init_gold_top10=0.5638 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2188 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9160 logit_acc_rollout_kept=0.9181 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=5001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 6000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=5100 epoch=5100/6000 epoch_step=1/1 micro_steps=5100 elapsed=35.1s lr=2.000000e-03 loss=0.2317 loss_recon=0.2317 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9266 corrupt_frac=1.0000 acc_corrupt=0.9266 loss_corrupt=0.2317 wrong_frac=0.7918 init_acc_corrupt=0.1488 acc_corrupt_t_0p0_0p2=0.8680 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6023 out_g_norm=0.2169 loss_all=0.1126 init_gold_top10=0.5863 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1946 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9649 logit_acc_rollout_kept=0.9625 +step=5200 epoch=5200/6000 epoch_step=1/1 micro_steps=5200 elapsed=34.5s lr=2.000000e-03 loss=0.2246 loss_recon=0.2246 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9292 corrupt_frac=1.0000 acc_corrupt=0.9292 loss_corrupt=0.2246 wrong_frac=0.7893 init_acc_corrupt=0.1506 acc_corrupt_t_0p0_0p2=0.8716 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.6165 out_g_norm=0.2112 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2049 init_gold_top10=0.6086 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1611 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9109 logit_acc_rollout_kept=0.9511 +step=5300 epoch=5300/6000 epoch_step=1/1 micro_steps=5300 elapsed=34.5s lr=2.000000e-03 loss=0.2264 loss_recon=0.2264 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9276 corrupt_frac=1.0000 acc_corrupt=0.9276 loss_corrupt=0.2264 wrong_frac=0.7934 init_acc_corrupt=0.1461 acc_corrupt_t_0p0_0p2=0.8718 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6305 out_g_norm=0.2050 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1943 init_gold_top10=0.5608 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1903 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9567 logit_acc_rollout_kept=0.9141 +step=5400 epoch=5400/6000 epoch_step=1/1 micro_steps=5400 elapsed=34.5s lr=2.000000e-03 loss=0.2296 loss_recon=0.2296 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9266 corrupt_frac=1.0000 acc_corrupt=0.9266 loss_corrupt=0.2296 wrong_frac=0.7914 init_acc_corrupt=0.1489 acc_corrupt_t_0p0_0p2=0.8682 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6508 out_g_norm=0.1937 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3028 init_gold_top10=0.5167 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1476 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8200 logit_acc_rollout_kept=0.9494 +step=5500 epoch=5500/6000 epoch_step=1/1 micro_steps=5500 elapsed=34.5s lr=2.000000e-03 loss=0.2268 loss_recon=0.2268 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9262 corrupt_frac=1.0000 acc_corrupt=0.9262 loss_corrupt=0.2268 wrong_frac=0.7930 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.8691 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9993 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.6615 out_g_norm=0.2104 loss_all=0.1531 init_gold_top10=0.6035 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1778 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9362 logit_acc_rollout_kept=0.9687 +step=5600 epoch=5600/6000 epoch_step=1/1 micro_steps=5600 elapsed=34.6s lr=2.000000e-03 loss=0.2183 loss_recon=0.2183 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9303 corrupt_frac=1.0000 acc_corrupt=0.9303 loss_corrupt=0.2183 wrong_frac=0.7925 init_acc_corrupt=0.1466 acc_corrupt_t_0p0_0p2=0.8759 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.6684 out_g_norm=0.1936 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2668 init_gold_top10=0.5649 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1581 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9056 logit_acc_rollout_kept=0.9404 +step=5700 epoch=5700/6000 epoch_step=1/1 micro_steps=5700 elapsed=34.6s lr=2.000000e-03 loss=0.2237 loss_recon=0.2237 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9303 corrupt_frac=1.0000 acc_corrupt=0.9303 loss_corrupt=0.2237 wrong_frac=0.7907 init_acc_corrupt=0.1496 acc_corrupt_t_0p0_0p2=0.8744 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6775 out_g_norm=0.2165 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1668 init_gold_top10=0.5983 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1918 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9848 logit_acc_rollout_kept=0.9038 +step=5800 epoch=5800/6000 epoch_step=1/1 micro_steps=5800 elapsed=34.6s lr=2.000000e-03 loss=0.2143 loss_recon=0.2143 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9320 corrupt_frac=1.0000 acc_corrupt=0.9320 loss_corrupt=0.2143 wrong_frac=0.7901 init_acc_corrupt=0.1496 acc_corrupt_t_0p0_0p2=0.8775 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6819 out_g_norm=0.1895 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1579 init_gold_top10=0.6291 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2105 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9585 logit_acc_rollout_kept=0.9492 +step=5900 epoch=5900/6000 epoch_step=1/1 micro_steps=5900 elapsed=34.6s lr=2.000000e-03 loss=0.2163 loss_recon=0.2163 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9324 corrupt_frac=1.0000 acc_corrupt=0.9324 loss_corrupt=0.2163 wrong_frac=0.7895 init_acc_corrupt=0.1512 acc_corrupt_t_0p0_0p2=0.8777 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6994 out_g_norm=0.2253 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3228 init_gold_top10=0.5883 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1821 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.8970 logit_acc_rollout_kept=0.9118 +step=6000 epoch=6000/6000 epoch_step=1/1 micro_steps=6000 elapsed=34.6s lr=2.000000e-03 loss=0.2194 loss_recon=0.2194 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9299 corrupt_frac=1.0000 acc_corrupt=0.9299 loss_corrupt=0.2194 wrong_frac=0.7888 init_acc_corrupt=0.1521 acc_corrupt_t_0p0_0p2=0.8730 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7132 out_g_norm=0.1901 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9982 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2391 init_gold_top10=0.5756 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2193 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9207 logit_acc_rollout_kept=0.9279 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=6001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 7000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=6100 epoch=6100/7000 epoch_step=1/1 micro_steps=6100 elapsed=35.0s lr=2.000000e-03 loss=0.2124 loss_recon=0.2124 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9332 corrupt_frac=1.0000 acc_corrupt=0.9332 loss_corrupt=0.2124 wrong_frac=0.7918 init_acc_corrupt=0.1489 acc_corrupt_t_0p0_0p2=0.8798 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7214 out_g_norm=0.1770 loss_all=0.1138 init_gold_top10=0.5794 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1949 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9471 logit_acc_rollout_kept=0.9692 +step=6200 epoch=6200/7000 epoch_step=1/1 micro_steps=6200 elapsed=34.4s lr=2.000000e-03 loss=0.2038 loss_recon=0.2038 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9360 corrupt_frac=1.0000 acc_corrupt=0.9360 loss_corrupt=0.2038 wrong_frac=0.7893 init_acc_corrupt=0.1508 acc_corrupt_t_0p0_0p2=0.8840 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7424 out_g_norm=0.2021 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2064 init_gold_top10=0.6067 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1611 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9039 logit_acc_rollout_kept=0.9594 +step=6300 epoch=6300/7000 epoch_step=1/1 micro_steps=6300 elapsed=34.4s lr=2.000000e-03 loss=0.2109 loss_recon=0.2109 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9325 corrupt_frac=1.0000 acc_corrupt=0.9325 loss_corrupt=0.2109 wrong_frac=0.7934 init_acc_corrupt=0.1462 acc_corrupt_t_0p0_0p2=0.8805 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7467 out_g_norm=0.1771 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1918 init_gold_top10=0.5600 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1896 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9278 logit_acc_rollout_kept=0.9398 +step=6400 epoch=6400/7000 epoch_step=1/1 micro_steps=6400 elapsed=34.4s lr=2.000000e-03 loss=0.2148 loss_recon=0.2148 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9328 corrupt_frac=1.0000 acc_corrupt=0.9328 loss_corrupt=0.2148 wrong_frac=0.7914 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.8792 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.7413 out_g_norm=0.1861 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2657 init_gold_top10=0.5238 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1479 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8514 logit_acc_rollout_kept=0.9383 +step=6500 epoch=6500/7000 epoch_step=1/1 micro_steps=6500 elapsed=34.4s lr=2.000000e-03 loss=0.2128 loss_recon=0.2128 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9316 corrupt_frac=1.0000 acc_corrupt=0.9316 loss_corrupt=0.2128 wrong_frac=0.7930 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.8786 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7445 out_g_norm=0.1659 loss_all=0.1465 init_gold_top10=0.6034 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1777 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9518 logit_acc_rollout_kept=0.9689 +step=6600 epoch=6600/7000 epoch_step=1/1 micro_steps=6600 elapsed=34.4s lr=2.000000e-03 loss=0.2062 loss_recon=0.2062 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9344 corrupt_frac=1.0000 acc_corrupt=0.9344 loss_corrupt=0.2062 wrong_frac=0.7925 init_acc_corrupt=0.1466 acc_corrupt_t_0p0_0p2=0.8833 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7419 out_g_norm=0.1691 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2827 init_gold_top10=0.5659 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1583 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8795 logit_acc_rollout_kept=0.9359 +step=6700 epoch=6700/7000 epoch_step=1/1 micro_steps=6700 elapsed=34.4s lr=2.000000e-03 loss=0.2117 loss_recon=0.2117 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9330 corrupt_frac=1.0000 acc_corrupt=0.9330 loss_corrupt=0.2117 wrong_frac=0.7907 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8794 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7395 out_g_norm=0.1868 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1578 init_gold_top10=0.5970 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1917 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9692 logit_acc_rollout_kept=0.9308 +step=6800 epoch=6800/7000 epoch_step=1/1 micro_steps=6800 elapsed=34.4s lr=2.000000e-03 loss=0.2005 loss_recon=0.2005 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9370 corrupt_frac=1.0000 acc_corrupt=0.9370 loss_corrupt=0.2005 wrong_frac=0.7901 init_acc_corrupt=0.1496 acc_corrupt_t_0p0_0p2=0.8865 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.7473 out_g_norm=0.1807 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1444 init_gold_top10=0.6314 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2104 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9567 logit_acc_rollout_kept=0.9669 +step=6900 epoch=6900/7000 epoch_step=1/1 micro_steps=6900 elapsed=34.4s lr=2.000000e-03 loss=0.2003 loss_recon=0.2003 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9366 corrupt_frac=1.0000 acc_corrupt=0.9366 loss_corrupt=0.2003 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.8851 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7528 out_g_norm=0.1809 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3084 init_gold_top10=0.5984 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1823 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9452 logit_acc_rollout_kept=0.9027 +step=7000 epoch=7000/7000 epoch_step=1/1 micro_steps=7000 elapsed=34.4s lr=2.000000e-03 loss=0.2047 loss_recon=0.2047 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9346 corrupt_frac=1.0000 acc_corrupt=0.9346 loss_corrupt=0.2047 wrong_frac=0.7888 init_acc_corrupt=0.1521 acc_corrupt_t_0p0_0p2=0.8815 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7552 out_g_norm=0.1642 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9992 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2338 init_gold_top10=0.5769 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2193 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9313 logit_acc_rollout_kept=0.9281 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=7001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 8000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=7100 epoch=7100/8000 epoch_step=1/1 micro_steps=7100 elapsed=35.1s lr=2.000000e-03 loss=0.2046 loss_recon=0.2046 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9357 corrupt_frac=1.0000 acc_corrupt=0.9357 loss_corrupt=0.2046 wrong_frac=0.7918 init_acc_corrupt=0.1489 acc_corrupt_t_0p0_0p2=0.8843 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7535 out_g_norm=0.1928 loss_all=0.1191 init_gold_top10=0.5836 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1947 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9552 logit_acc_rollout_kept=0.9609 +step=7200 epoch=7200/8000 epoch_step=1/1 micro_steps=7200 elapsed=34.4s lr=2.000000e-03 loss=0.2005 loss_recon=0.2005 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9372 corrupt_frac=1.0000 acc_corrupt=0.9372 loss_corrupt=0.2005 wrong_frac=0.7893 init_acc_corrupt=0.1507 acc_corrupt_t_0p0_0p2=0.8862 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7541 out_g_norm=0.1751 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2179 init_gold_top10=0.5967 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1611 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.8876 logit_acc_rollout_kept=0.9631 +step=7300 epoch=7300/8000 epoch_step=1/1 micro_steps=7300 elapsed=34.4s lr=2.000000e-03 loss=0.1968 loss_recon=0.1968 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9374 corrupt_frac=1.0000 acc_corrupt=0.9374 loss_corrupt=0.1968 wrong_frac=0.7934 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.8891 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7662 out_g_norm=0.1701 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1601 init_gold_top10=0.5604 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1907 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9525 logit_acc_rollout_kept=0.9563 +step=7400 epoch=7400/8000 epoch_step=1/1 micro_steps=7400 elapsed=34.4s lr=2.000000e-03 loss=0.2025 loss_recon=0.2025 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9362 corrupt_frac=1.0000 acc_corrupt=0.9362 loss_corrupt=0.2025 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.8853 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.7659 out_g_norm=0.1715 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3038 init_gold_top10=0.5147 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1482 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8528 logit_acc_rollout_kept=0.9490 +step=7500 epoch=7500/8000 epoch_step=1/1 micro_steps=7500 elapsed=34.4s lr=2.000000e-03 loss=0.2076 loss_recon=0.2076 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9342 corrupt_frac=1.0000 acc_corrupt=0.9342 loss_corrupt=0.2076 wrong_frac=0.7930 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.8832 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7559 out_g_norm=0.1788 loss_all=0.1559 init_gold_top10=0.6092 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1777 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9420 logit_acc_rollout_kept=0.9570 +step=7600 epoch=7600/8000 epoch_step=1/1 micro_steps=7600 elapsed=34.4s lr=2.000000e-03 loss=0.1901 loss_recon=0.1901 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9390 corrupt_frac=1.0000 acc_corrupt=0.9390 loss_corrupt=0.1901 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.8915 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7551 out_g_norm=0.1646 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2306 init_gold_top10=0.5678 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1584 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8945 logit_acc_rollout_kept=0.9606 +step=7700 epoch=7700/8000 epoch_step=1/1 micro_steps=7700 elapsed=34.4s lr=2.000000e-03 loss=0.2019 loss_recon=0.2019 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9358 corrupt_frac=1.0000 acc_corrupt=0.9358 loss_corrupt=0.2019 wrong_frac=0.7907 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8844 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7515 out_g_norm=0.1645 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1493 init_gold_top10=0.5988 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1918 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9626 logit_acc_rollout_kept=0.9310 +step=7800 epoch=7800/8000 epoch_step=1/1 micro_steps=7800 elapsed=34.4s lr=2.000000e-03 loss=0.1973 loss_recon=0.1973 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9375 corrupt_frac=1.0000 acc_corrupt=0.9375 loss_corrupt=0.1973 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8874 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.7433 out_g_norm=0.1703 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2014 init_gold_top10=0.6239 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2106 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9346 logit_acc_rollout_kept=0.9517 +step=7900 epoch=7900/8000 epoch_step=1/1 micro_steps=7900 elapsed=34.4s lr=2.000000e-03 loss=0.1879 loss_recon=0.1879 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9406 corrupt_frac=1.0000 acc_corrupt=0.9406 loss_corrupt=0.1879 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.8925 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7452 out_g_norm=0.1573 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3429 init_gold_top10=0.5950 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1816 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.8933 logit_acc_rollout_kept=0.9201 +step=8000 epoch=8000/8000 epoch_step=1/1 micro_steps=8000 elapsed=34.4s lr=2.000000e-03 loss=0.1900 loss_recon=0.1900 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9389 corrupt_frac=1.0000 acc_corrupt=0.9389 loss_corrupt=0.1900 wrong_frac=0.7888 init_acc_corrupt=0.1522 acc_corrupt_t_0p0_0p2=0.8893 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7504 out_g_norm=0.1562 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1712 init_gold_top10=0.5842 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2193 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9272 logit_acc_rollout_kept=0.9514 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=8001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 9000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=8100 epoch=8100/9000 epoch_step=1/1 micro_steps=8100 elapsed=35.0s lr=2.000000e-03 loss=0.1922 loss_recon=0.1922 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9388 corrupt_frac=1.0000 acc_corrupt=0.9388 loss_corrupt=0.1922 wrong_frac=0.7918 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.8899 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7459 out_g_norm=0.1660 loss_all=0.1450 init_gold_top10=0.5799 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1946 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9445 logit_acc_rollout_kept=0.9525 +step=8200 epoch=8200/9000 epoch_step=1/1 micro_steps=8200 elapsed=34.4s lr=2.000000e-03 loss=0.1808 loss_recon=0.1808 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9428 corrupt_frac=1.0000 acc_corrupt=0.9428 loss_corrupt=0.1808 wrong_frac=0.7893 init_acc_corrupt=0.1508 acc_corrupt_t_0p0_0p2=0.8964 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7536 out_g_norm=0.1614 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2269 init_gold_top10=0.5974 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1610 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.8932 logit_acc_rollout_kept=0.9548 +step=8300 epoch=8300/9000 epoch_step=1/1 micro_steps=8300 elapsed=34.4s lr=2.000000e-03 loss=0.1989 loss_recon=0.1989 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9374 corrupt_frac=1.0000 acc_corrupt=0.9374 loss_corrupt=0.1989 wrong_frac=0.7934 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.8891 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7419 out_g_norm=0.1731 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1699 init_gold_top10=0.5570 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1903 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9681 logit_acc_rollout_kept=0.9293 +step=8400 epoch=8400/9000 epoch_step=1/1 micro_steps=8400 elapsed=34.4s lr=2.000000e-03 loss=0.1923 loss_recon=0.1923 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9393 corrupt_frac=1.0000 acc_corrupt=0.9393 loss_corrupt=0.1923 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.8909 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.7300 out_g_norm=0.1532 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3777 init_gold_top10=0.5077 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1478 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8540 logit_acc_rollout_kept=0.9429 +step=8500 epoch=8500/9000 epoch_step=1/1 micro_steps=8500 elapsed=34.4s lr=2.000000e-03 loss=0.1932 loss_recon=0.1932 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9385 corrupt_frac=1.0000 acc_corrupt=0.9385 loss_corrupt=0.1932 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.8908 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7284 out_g_norm=0.1581 loss_all=0.2109 init_gold_top10=0.6005 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1776 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9265 logit_acc_rollout_kept=0.9424 +step=8600 epoch=8600/9000 epoch_step=1/1 micro_steps=8600 elapsed=34.4s lr=2.000000e-03 loss=0.1862 loss_recon=0.1862 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9426 corrupt_frac=1.0000 acc_corrupt=0.9426 loss_corrupt=0.1862 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.8979 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7246 out_g_norm=0.1683 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2296 init_gold_top10=0.5666 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8895 logit_acc_rollout_kept=0.9602 +step=8700 epoch=8700/9000 epoch_step=1/1 micro_steps=8700 elapsed=34.4s lr=2.000000e-03 loss=0.1915 loss_recon=0.1915 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9394 corrupt_frac=1.0000 acc_corrupt=0.9394 loss_corrupt=0.1915 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.8909 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7287 out_g_norm=0.1625 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1677 init_gold_top10=0.5967 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1917 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9639 logit_acc_rollout_kept=0.9227 +step=8800 epoch=8800/9000 epoch_step=1/1 micro_steps=8800 elapsed=34.4s lr=2.000000e-03 loss=0.1823 loss_recon=0.1823 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9430 corrupt_frac=1.0000 acc_corrupt=0.9430 loss_corrupt=0.1823 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8973 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.7285 out_g_norm=0.1699 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1869 init_gold_top10=0.6252 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2108 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9411 logit_acc_rollout_kept=0.9646 +step=8900 epoch=8900/9000 epoch_step=1/1 micro_steps=8900 elapsed=34.4s lr=2.000000e-03 loss=0.1873 loss_recon=0.1873 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9399 corrupt_frac=1.0000 acc_corrupt=0.9399 loss_corrupt=0.1873 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.8911 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7230 out_g_norm=0.1715 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2217 init_gold_top10=0.6066 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1826 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9415 logit_acc_rollout_kept=0.9293 +step=9000 epoch=9000/9000 epoch_step=1/1 micro_steps=9000 elapsed=34.4s lr=2.000000e-03 loss=0.1858 loss_recon=0.1858 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9407 corrupt_frac=1.0000 acc_corrupt=0.9407 loss_corrupt=0.1858 wrong_frac=0.7888 init_acc_corrupt=0.1521 acc_corrupt_t_0p0_0p2=0.8926 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7256 out_g_norm=0.1685 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1354 init_gold_top10=0.5809 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2191 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9533 logit_acc_rollout_kept=0.9538 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=9001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 10000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=9100 epoch=9100/10000 epoch_step=1/1 micro_steps=9100 elapsed=35.1s lr=2.000000e-03 loss=0.1877 loss_recon=0.1877 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9407 corrupt_frac=1.0000 acc_corrupt=0.9407 loss_corrupt=0.1877 wrong_frac=0.7918 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.8933 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7347 out_g_norm=0.1578 loss_all=0.1170 init_gold_top10=0.5847 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1943 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9599 logit_acc_rollout_kept=0.9631 +step=9200 epoch=9200/10000 epoch_step=1/1 micro_steps=9200 elapsed=34.4s lr=2.000000e-03 loss=0.1760 loss_recon=0.1760 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9444 corrupt_frac=1.0000 acc_corrupt=0.9444 loss_corrupt=0.1760 wrong_frac=0.7893 init_acc_corrupt=0.1508 acc_corrupt_t_0p0_0p2=0.8993 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7361 out_g_norm=0.1516 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1619 init_gold_top10=0.6116 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1610 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9280 logit_acc_rollout_kept=0.9719 +step=9300 epoch=9300/10000 epoch_step=1/1 micro_steps=9300 elapsed=34.4s lr=2.000000e-03 loss=0.1870 loss_recon=0.1870 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9413 corrupt_frac=1.0000 acc_corrupt=0.9413 loss_corrupt=0.1870 wrong_frac=0.7934 init_acc_corrupt=0.1462 acc_corrupt_t_0p0_0p2=0.8960 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7379 out_g_norm=0.1649 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1320 init_gold_top10=0.5572 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1907 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9665 logit_acc_rollout_kept=0.9566 +step=9400 epoch=9400/10000 epoch_step=1/1 micro_steps=9400 elapsed=34.4s lr=2.000000e-03 loss=0.1850 loss_recon=0.1850 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9419 corrupt_frac=1.0000 acc_corrupt=0.9419 loss_corrupt=0.1850 wrong_frac=0.7914 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.8956 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.7361 out_g_norm=0.1468 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3255 init_gold_top10=0.5134 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1477 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8254 logit_acc_rollout_kept=0.9368 +step=9500 epoch=9500/10000 epoch_step=1/1 micro_steps=9500 elapsed=34.4s lr=2.000000e-03 loss=0.1826 loss_recon=0.1826 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9422 corrupt_frac=1.0000 acc_corrupt=0.9422 loss_corrupt=0.1826 wrong_frac=0.7930 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.8975 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7368 out_g_norm=0.1500 loss_all=0.1732 init_gold_top10=0.6082 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1779 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9359 logit_acc_rollout_kept=0.9512 +step=9600 epoch=9600/10000 epoch_step=1/1 micro_steps=9600 elapsed=34.4s lr=2.000000e-03 loss=0.1743 loss_recon=0.1743 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9450 corrupt_frac=1.0000 acc_corrupt=0.9450 loss_corrupt=0.1743 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.9022 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7346 out_g_norm=0.1496 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2262 init_gold_top10=0.5680 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8957 logit_acc_rollout_kept=0.9543 +step=9700 epoch=9700/10000 epoch_step=1/1 micro_steps=9700 elapsed=34.4s lr=2.000000e-03 loss=0.1879 loss_recon=0.1879 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9412 corrupt_frac=1.0000 acc_corrupt=0.9412 loss_corrupt=0.1879 wrong_frac=0.7907 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8941 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7268 out_g_norm=0.1594 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1719 init_gold_top10=0.5915 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1916 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9528 logit_acc_rollout_kept=0.9263 +step=9800 epoch=9800/10000 epoch_step=1/1 micro_steps=9800 elapsed=34.4s lr=2.000000e-03 loss=0.1774 loss_recon=0.1774 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9440 corrupt_frac=1.0000 acc_corrupt=0.9440 loss_corrupt=0.1774 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8990 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.7214 out_g_norm=0.1531 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1233 init_gold_top10=0.6316 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2107 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9406 logit_acc_rollout_kept=0.9758 +step=9900 epoch=9900/10000 epoch_step=1/1 micro_steps=9900 elapsed=34.4s lr=2.000000e-03 loss=0.1855 loss_recon=0.1855 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9410 corrupt_frac=1.0000 acc_corrupt=0.9410 loss_corrupt=0.1855 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.8931 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7243 out_g_norm=0.1600 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2158 init_gold_top10=0.6030 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1828 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9527 logit_acc_rollout_kept=0.9121 +step=10000 epoch=10000/10000 epoch_step=1/1 micro_steps=10000 elapsed=34.4s lr=2.000000e-03 loss=0.1817 loss_recon=0.1817 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9412 corrupt_frac=1.0000 acc_corrupt=0.9412 loss_corrupt=0.1817 wrong_frac=0.7888 init_acc_corrupt=0.1521 acc_corrupt_t_0p0_0p2=0.8935 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7185 out_g_norm=0.1431 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1727 init_gold_top10=0.5764 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2193 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9399 logit_acc_rollout_kept=0.9314 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=10001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 11000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=10100 epoch=10100/11000 epoch_step=1/1 micro_steps=10100 elapsed=35.0s lr=2.000000e-03 loss=0.1749 loss_recon=0.1749 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9447 corrupt_frac=1.0000 acc_corrupt=0.9447 loss_corrupt=0.1749 wrong_frac=0.7918 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.9005 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7208 out_g_norm=0.1589 loss_all=0.1037 init_gold_top10=0.5851 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1949 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9650 logit_acc_rollout_kept=0.9581 +step=10200 epoch=10200/11000 epoch_step=1/1 micro_steps=10200 elapsed=34.4s lr=2.000000e-03 loss=0.1717 loss_recon=0.1717 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9459 corrupt_frac=1.0000 acc_corrupt=0.9459 loss_corrupt=0.1717 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9019 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7297 out_g_norm=0.1499 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2924 init_gold_top10=0.5881 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1605 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.8398 logit_acc_rollout_kept=0.9690 +step=10300 epoch=10300/11000 epoch_step=1/1 micro_steps=10300 elapsed=34.4s lr=2.000000e-03 loss=0.1863 loss_recon=0.1863 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9412 corrupt_frac=1.0000 acc_corrupt=0.9412 loss_corrupt=0.1863 wrong_frac=0.7934 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.8959 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7253 out_g_norm=0.1560 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2569 init_gold_top10=0.5572 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1903 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9318 logit_acc_rollout_kept=0.9170 +step=10400 epoch=10400/11000 epoch_step=1/1 micro_steps=10400 elapsed=34.4s lr=2.000000e-03 loss=0.1813 loss_recon=0.1813 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9421 corrupt_frac=1.0000 acc_corrupt=0.9421 loss_corrupt=0.1813 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.8959 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.7245 out_g_norm=0.1462 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.3059 init_gold_top10=0.5194 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1482 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8579 logit_acc_rollout_kept=0.9490 +step=10500 epoch=10500/11000 epoch_step=1/1 micro_steps=10500 elapsed=34.4s lr=2.000000e-03 loss=0.1775 loss_recon=0.1775 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9430 corrupt_frac=1.0000 acc_corrupt=0.9430 loss_corrupt=0.1775 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.8988 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7284 out_g_norm=0.1520 loss_all=0.1302 init_gold_top10=0.5975 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1778 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9408 logit_acc_rollout_kept=0.9843 +step=10600 epoch=10600/11000 epoch_step=1/1 micro_steps=10600 elapsed=34.4s lr=2.000000e-03 loss=0.1711 loss_recon=0.1711 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9463 corrupt_frac=1.0000 acc_corrupt=0.9463 loss_corrupt=0.1711 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.9044 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7263 out_g_norm=0.1568 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1840 init_gold_top10=0.5779 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1585 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9204 logit_acc_rollout_kept=0.9713 +step=10700 epoch=10700/11000 epoch_step=1/1 micro_steps=10700 elapsed=34.4s lr=2.000000e-03 loss=0.1779 loss_recon=0.1779 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9431 corrupt_frac=1.0000 acc_corrupt=0.9431 loss_corrupt=0.1779 wrong_frac=0.7907 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.8975 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7234 out_g_norm=0.1507 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1220 init_gold_top10=0.5913 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1918 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9683 logit_acc_rollout_kept=0.9570 +step=10800 epoch=10800/11000 epoch_step=1/1 micro_steps=10800 elapsed=34.4s lr=2.000000e-03 loss=0.1697 loss_recon=0.1697 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9469 corrupt_frac=1.0000 acc_corrupt=0.9469 loss_corrupt=0.1697 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.9044 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.7228 out_g_norm=0.1383 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0947 init_gold_top10=0.6332 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2106 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9593 logit_acc_rollout_kept=0.9779 +step=10900 epoch=10900/11000 epoch_step=1/1 micro_steps=10900 elapsed=34.4s lr=2.000000e-03 loss=0.1746 loss_recon=0.1746 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9440 corrupt_frac=1.0000 acc_corrupt=0.9440 loss_corrupt=0.1746 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.8986 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7208 out_g_norm=0.1592 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2467 init_gold_top10=0.5942 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1825 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9120 logit_acc_rollout_kept=0.9310 +step=11000 epoch=11000/11000 epoch_step=1/1 micro_steps=11000 elapsed=34.4s lr=2.000000e-03 loss=0.1811 loss_recon=0.1811 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9429 corrupt_frac=1.0000 acc_corrupt=0.9429 loss_corrupt=0.1811 wrong_frac=0.7888 init_acc_corrupt=0.1521 acc_corrupt_t_0p0_0p2=0.8965 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.7142 out_g_norm=0.1419 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9986 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1857 init_gold_top10=0.5831 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2196 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9406 logit_acc_rollout_kept=0.9494 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=11001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 12000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=11100 epoch=11100/12000 epoch_step=1/1 micro_steps=11100 elapsed=35.1s lr=2.000000e-03 loss=0.1763 loss_recon=0.1763 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9449 corrupt_frac=1.0000 acc_corrupt=0.9449 loss_corrupt=0.1763 wrong_frac=0.7918 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.9009 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7030 out_g_norm=0.1576 loss_all=0.0985 init_gold_top10=0.5847 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1950 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9709 logit_acc_rollout_kept=0.9697 +step=11200 epoch=11200/12000 epoch_step=1/1 micro_steps=11200 elapsed=34.4s lr=2.000000e-03 loss=0.1702 loss_recon=0.1702 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9466 corrupt_frac=1.0000 acc_corrupt=0.9466 loss_corrupt=0.1702 wrong_frac=0.7893 init_acc_corrupt=0.1507 acc_corrupt_t_0p0_0p2=0.9032 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7044 out_g_norm=0.1537 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1731 init_gold_top10=0.6041 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1614 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.8986 logit_acc_rollout_kept=0.9784 +step=11300 epoch=11300/12000 epoch_step=1/1 micro_steps=11300 elapsed=34.4s lr=2.000000e-03 loss=0.1780 loss_recon=0.1780 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9435 corrupt_frac=1.0000 acc_corrupt=0.9435 loss_corrupt=0.1780 wrong_frac=0.7934 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.8998 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7091 out_g_norm=0.1562 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1648 init_gold_top10=0.5575 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1897 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9473 logit_acc_rollout_kept=0.9472 +step=11400 epoch=11400/12000 epoch_step=1/1 micro_steps=11400 elapsed=34.4s lr=2.000000e-03 loss=0.1707 loss_recon=0.1707 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9467 corrupt_frac=1.0000 acc_corrupt=0.9467 loss_corrupt=0.1707 wrong_frac=0.7914 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.9042 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6990 out_g_norm=0.1430 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2340 init_gold_top10=0.5150 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1482 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8877 logit_acc_rollout_kept=0.9515 +step=11500 epoch=11500/12000 epoch_step=1/1 micro_steps=11500 elapsed=34.4s lr=2.000000e-03 loss=0.1702 loss_recon=0.1702 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9458 corrupt_frac=1.0000 acc_corrupt=0.9458 loss_corrupt=0.1702 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.9037 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7034 out_g_norm=0.1474 loss_all=0.1460 init_gold_top10=0.5940 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1777 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9277 logit_acc_rollout_kept=0.9747 +step=11600 epoch=11600/12000 epoch_step=1/1 micro_steps=11600 elapsed=34.4s lr=2.000000e-03 loss=0.1682 loss_recon=0.1682 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9465 corrupt_frac=1.0000 acc_corrupt=0.9465 loss_corrupt=0.1682 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.9048 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7058 out_g_norm=0.1500 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1696 init_gold_top10=0.5698 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8951 logit_acc_rollout_kept=0.9852 +step=11700 epoch=11700/12000 epoch_step=1/1 micro_steps=11700 elapsed=34.4s lr=2.000000e-03 loss=0.1704 loss_recon=0.1704 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9457 corrupt_frac=1.0000 acc_corrupt=0.9457 loss_corrupt=0.1704 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9022 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.7037 out_g_norm=0.1493 acc_corrupt_t_0p8_1p0=0.9980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1562 init_gold_top10=0.5921 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1918 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9660 logit_acc_rollout_kept=0.9383 +step=11800 epoch=11800/12000 epoch_step=1/1 micro_steps=11800 elapsed=34.4s lr=2.000000e-03 loss=0.1678 loss_recon=0.1678 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9476 corrupt_frac=1.0000 acc_corrupt=0.9476 loss_corrupt=0.1678 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.9055 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6999 out_g_norm=0.1571 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9941 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2027 init_gold_top10=0.6180 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2108 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9373 logit_acc_rollout_kept=0.9563 +step=11900 epoch=11900/12000 epoch_step=1/1 micro_steps=11900 elapsed=34.4s lr=2.000000e-03 loss=0.1725 loss_recon=0.1725 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9438 corrupt_frac=1.0000 acc_corrupt=0.9438 loss_corrupt=0.1725 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.8982 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.7001 out_g_norm=0.1537 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2308 init_gold_top10=0.5992 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1829 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9227 logit_acc_rollout_kept=0.9133 +step=12000 epoch=12000/12000 epoch_step=1/1 micro_steps=12000 elapsed=34.4s lr=2.000000e-03 loss=0.1743 loss_recon=0.1743 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9463 corrupt_frac=1.0000 acc_corrupt=0.9463 loss_corrupt=0.1743 wrong_frac=0.7888 init_acc_corrupt=0.1522 acc_corrupt_t_0p0_0p2=0.9027 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6957 out_g_norm=0.1560 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1174 init_gold_top10=0.5845 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2195 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9604 logit_acc_rollout_kept=0.9641 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=12001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 13000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=12100 epoch=12100/13000 epoch_step=1/1 micro_steps=12100 elapsed=35.1s lr=2.000000e-03 loss=0.1716 loss_recon=0.1716 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9456 corrupt_frac=1.0000 acc_corrupt=0.9456 loss_corrupt=0.1716 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9021 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6920 out_g_norm=0.1396 loss_all=0.0966 init_gold_top10=0.5868 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1951 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9742 logit_acc_rollout_kept=0.9647 +step=12200 epoch=12200/13000 epoch_step=1/1 micro_steps=12200 elapsed=34.4s lr=2.000000e-03 loss=0.1605 loss_recon=0.1605 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9484 corrupt_frac=1.0000 acc_corrupt=0.9484 loss_corrupt=0.1605 wrong_frac=0.7893 init_acc_corrupt=0.1508 acc_corrupt_t_0p0_0p2=0.9064 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.7004 out_g_norm=0.1389 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1528 init_gold_top10=0.6075 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1612 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9320 logit_acc_rollout_kept=0.9706 +step=12300 epoch=12300/13000 epoch_step=1/1 micro_steps=12300 elapsed=34.4s lr=2.000000e-03 loss=0.1727 loss_recon=0.1727 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9455 corrupt_frac=1.0000 acc_corrupt=0.9455 loss_corrupt=0.1727 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9035 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.7036 out_g_norm=0.1499 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1296 init_gold_top10=0.5570 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1906 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9525 logit_acc_rollout_kept=0.9560 +step=12400 epoch=12400/13000 epoch_step=1/1 micro_steps=12400 elapsed=34.4s lr=2.000000e-03 loss=0.1698 loss_recon=0.1698 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9465 corrupt_frac=1.0000 acc_corrupt=0.9465 loss_corrupt=0.1698 wrong_frac=0.7914 init_acc_corrupt=0.1490 acc_corrupt_t_0p0_0p2=0.9038 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6965 out_g_norm=0.1344 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2454 init_gold_top10=0.5201 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1475 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8877 logit_acc_rollout_kept=0.9572 +step=12500 epoch=12500/13000 epoch_step=1/1 micro_steps=12500 elapsed=34.4s lr=2.000000e-03 loss=0.1678 loss_recon=0.1678 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9462 corrupt_frac=1.0000 acc_corrupt=0.9462 loss_corrupt=0.1678 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.9045 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.7044 out_g_norm=0.1388 loss_all=0.1650 init_gold_top10=0.5986 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1774 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9179 logit_acc_rollout_kept=0.9794 +step=12600 epoch=12600/13000 epoch_step=1/1 micro_steps=12600 elapsed=34.4s lr=2.000000e-03 loss=0.1612 loss_recon=0.1612 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9483 corrupt_frac=1.0000 acc_corrupt=0.9483 loss_corrupt=0.1612 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.9079 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.7037 out_g_norm=0.1336 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2092 init_gold_top10=0.5687 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1585 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8949 logit_acc_rollout_kept=0.9649 +step=12700 epoch=12700/13000 epoch_step=1/1 micro_steps=12700 elapsed=34.4s lr=2.000000e-03 loss=0.1712 loss_recon=0.1712 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9463 corrupt_frac=1.0000 acc_corrupt=0.9463 loss_corrupt=0.1712 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9032 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6934 out_g_norm=0.1472 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1089 init_gold_top10=0.5960 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1919 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9648 logit_acc_rollout_kept=0.9666 +step=12800 epoch=12800/13000 epoch_step=1/1 micro_steps=12800 elapsed=34.4s lr=2.000000e-03 loss=0.1646 loss_recon=0.1646 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9491 corrupt_frac=1.0000 acc_corrupt=0.9491 loss_corrupt=0.1646 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.9082 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6809 out_g_norm=0.1524 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1967 init_gold_top10=0.6181 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2105 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9342 logit_acc_rollout_kept=0.9875 +step=12900 epoch=12900/13000 epoch_step=1/1 micro_steps=12900 elapsed=34.4s lr=2.000000e-03 loss=0.1682 loss_recon=0.1682 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9463 corrupt_frac=1.0000 acc_corrupt=0.9463 loss_corrupt=0.1682 wrong_frac=0.7895 init_acc_corrupt=0.1513 acc_corrupt_t_0p0_0p2=0.9028 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6724 out_g_norm=0.1454 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2047 init_gold_top10=0.6058 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1826 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9338 logit_acc_rollout_kept=0.9418 +step=13000 epoch=13000/13000 epoch_step=1/1 micro_steps=13000 elapsed=34.4s lr=2.000000e-03 loss=0.1643 loss_recon=0.1643 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9473 corrupt_frac=1.0000 acc_corrupt=0.9473 loss_corrupt=0.1643 wrong_frac=0.7888 init_acc_corrupt=0.1522 acc_corrupt_t_0p0_0p2=0.9046 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6824 out_g_norm=0.1396 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9994 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2197 init_gold_top10=0.5756 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2188 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9317 logit_acc_rollout_kept=0.9334 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=13001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 14000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=13100 epoch=13100/14000 epoch_step=1/1 micro_steps=13100 elapsed=35.1s lr=2.000000e-03 loss=0.1687 loss_recon=0.1687 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9468 corrupt_frac=1.0000 acc_corrupt=0.9468 loss_corrupt=0.1687 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9042 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6948 out_g_norm=0.1356 loss_all=0.0947 init_gold_top10=0.5858 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1948 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9672 logit_acc_rollout_kept=0.9717 +step=13200 epoch=13200/14000 epoch_step=1/1 micro_steps=13200 elapsed=36.6s lr=2.000000e-03 loss=0.1537 loss_recon=0.1537 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9522 corrupt_frac=1.0000 acc_corrupt=0.9522 loss_corrupt=0.1537 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9134 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.6902 out_g_norm=0.1511 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1198 init_gold_top10=0.6143 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1612 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9419 logit_acc_rollout_kept=0.9837 +step=13300 epoch=13300/14000 epoch_step=1/1 micro_steps=13300 elapsed=44.6s lr=2.000000e-03 loss=0.1639 loss_recon=0.1639 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9484 corrupt_frac=1.0000 acc_corrupt=0.9484 loss_corrupt=0.1639 wrong_frac=0.7934 init_acc_corrupt=0.1463 acc_corrupt_t_0p0_0p2=0.9086 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6949 out_g_norm=0.1441 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1301 init_gold_top10=0.5573 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1906 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9472 logit_acc_rollout_kept=0.9664 +step=13400 epoch=13400/14000 epoch_step=1/1 micro_steps=13400 elapsed=36.2s lr=2.000000e-03 loss=0.1676 loss_recon=0.1676 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9469 corrupt_frac=1.0000 acc_corrupt=0.9469 loss_corrupt=0.1676 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9046 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6895 out_g_norm=0.1466 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2151 init_gold_top10=0.5271 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1480 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8851 logit_acc_rollout_kept=0.9534 +step=13500 epoch=13500/14000 epoch_step=1/1 micro_steps=13500 elapsed=59.0s lr=2.000000e-03 loss=0.1607 loss_recon=0.1607 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9484 corrupt_frac=1.0000 acc_corrupt=0.9484 loss_corrupt=0.1607 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.9084 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.6907 out_g_norm=0.1386 loss_all=0.1666 init_gold_top10=0.6030 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1777 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9174 logit_acc_rollout_kept=0.9612 +step=13600 epoch=13600/14000 epoch_step=1/1 micro_steps=13600 elapsed=35.8s lr=2.000000e-03 loss=0.1521 loss_recon=0.1521 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9513 corrupt_frac=1.0000 acc_corrupt=0.9513 loss_corrupt=0.1521 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9133 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.6890 out_g_norm=0.1408 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1895 init_gold_top10=0.5783 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9099 logit_acc_rollout_kept=0.9616 +step=13700 epoch=13700/14000 epoch_step=1/1 micro_steps=13700 elapsed=35.3s lr=2.000000e-03 loss=0.1647 loss_recon=0.1647 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9476 corrupt_frac=1.0000 acc_corrupt=0.9476 loss_corrupt=0.1647 wrong_frac=0.7907 init_acc_corrupt=0.1499 acc_corrupt_t_0p0_0p2=0.9055 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6811 out_g_norm=0.1420 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1592 init_gold_top10=0.5924 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1920 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9705 logit_acc_rollout_kept=0.9259 +step=13800 epoch=13800/14000 epoch_step=1/1 micro_steps=13800 elapsed=34.4s lr=2.000000e-03 loss=0.1556 loss_recon=0.1556 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9496 corrupt_frac=1.0000 acc_corrupt=0.9496 loss_corrupt=0.1556 wrong_frac=0.7901 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9092 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6774 out_g_norm=0.1333 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1797 init_gold_top10=0.6287 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2108 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9387 logit_acc_rollout_kept=0.9654 +step=13900 epoch=13900/14000 epoch_step=1/1 micro_steps=13900 elapsed=34.4s lr=2.000000e-03 loss=0.1603 loss_recon=0.1603 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9480 corrupt_frac=1.0000 acc_corrupt=0.9480 loss_corrupt=0.1603 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.9059 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6696 out_g_norm=0.1345 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1595 init_gold_top10=0.6057 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1826 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9437 logit_acc_rollout_kept=0.9508 +step=14000 epoch=14000/14000 epoch_step=1/1 micro_steps=14000 elapsed=34.4s lr=2.000000e-03 loss=0.1632 loss_recon=0.1632 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9478 corrupt_frac=1.0000 acc_corrupt=0.9478 loss_corrupt=0.1632 wrong_frac=0.7888 init_acc_corrupt=0.1523 acc_corrupt_t_0p0_0p2=0.9054 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6620 out_g_norm=0.1337 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9982 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1820 init_gold_top10=0.5755 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2192 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9200 logit_acc_rollout_kept=0.9546 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=14001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 15000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=14100 epoch=14100/15000 epoch_step=1/1 micro_steps=14100 elapsed=35.1s lr=2.000000e-03 loss=0.1647 loss_recon=0.1647 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9471 corrupt_frac=1.0000 acc_corrupt=0.9471 loss_corrupt=0.1647 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9048 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6558 out_g_norm=0.1423 loss_all=0.0858 init_gold_top10=0.5866 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1947 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9756 logit_acc_rollout_kept=0.9689 +step=14200 epoch=14200/15000 epoch_step=1/1 micro_steps=14200 elapsed=34.4s lr=2.000000e-03 loss=0.1505 loss_recon=0.1505 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9520 corrupt_frac=1.0000 acc_corrupt=0.9520 loss_corrupt=0.1505 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9131 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.6545 out_g_norm=0.1367 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1478 init_gold_top10=0.6060 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1604 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9224 logit_acc_rollout_kept=0.9839 +step=14300 epoch=14300/15000 epoch_step=1/1 micro_steps=14300 elapsed=34.4s lr=2.000000e-03 loss=0.1599 loss_recon=0.1599 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9490 corrupt_frac=1.0000 acc_corrupt=0.9490 loss_corrupt=0.1599 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9096 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6417 out_g_norm=0.1360 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1999 init_gold_top10=0.5570 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1901 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9282 logit_acc_rollout_kept=0.9427 +step=14400 epoch=14400/15000 epoch_step=1/1 micro_steps=14400 elapsed=34.4s lr=2.000000e-03 loss=0.1571 loss_recon=0.1571 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9495 corrupt_frac=1.0000 acc_corrupt=0.9495 loss_corrupt=0.1571 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9092 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6415 out_g_norm=0.1339 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2079 init_gold_top10=0.5203 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1480 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8940 logit_acc_rollout_kept=0.9633 +step=14500 epoch=14500/15000 epoch_step=1/1 micro_steps=14500 elapsed=34.4s lr=2.000000e-03 loss=0.1620 loss_recon=0.1620 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9478 corrupt_frac=1.0000 acc_corrupt=0.9478 loss_corrupt=0.1620 wrong_frac=0.7930 init_acc_corrupt=0.1469 acc_corrupt_t_0p0_0p2=0.9073 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.6437 out_g_norm=0.1340 loss_all=0.1596 init_gold_top10=0.5932 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1779 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9252 logit_acc_rollout_kept=0.9896 +step=14600 epoch=14600/15000 epoch_step=1/1 micro_steps=14600 elapsed=34.4s lr=2.000000e-03 loss=0.1610 loss_recon=0.1610 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9484 corrupt_frac=1.0000 acc_corrupt=0.9484 loss_corrupt=0.1610 wrong_frac=0.7925 init_acc_corrupt=0.1467 acc_corrupt_t_0p0_0p2=0.9083 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.6339 out_g_norm=0.1486 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1458 init_gold_top10=0.5784 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1587 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9072 logit_acc_rollout_kept=0.9940 +step=14700 epoch=14700/15000 epoch_step=1/1 micro_steps=14700 elapsed=34.4s lr=2.000000e-03 loss=0.1608 loss_recon=0.1608 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9485 corrupt_frac=1.0000 acc_corrupt=0.9485 loss_corrupt=0.1608 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9073 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6320 out_g_norm=0.1258 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1417 init_gold_top10=0.5935 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1918 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9693 logit_acc_rollout_kept=0.9270 +step=14800 epoch=14800/15000 epoch_step=1/1 micro_steps=14800 elapsed=82.4s lr=2.000000e-03 loss=0.1526 loss_recon=0.1526 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9509 corrupt_frac=1.0000 acc_corrupt=0.9509 loss_corrupt=0.1526 wrong_frac=0.7901 init_acc_corrupt=0.1497 acc_corrupt_t_0p0_0p2=0.9115 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6409 out_g_norm=0.1435 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9893 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1932 init_gold_top10=0.6277 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2109 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9309 logit_acc_rollout_kept=0.9454 +step=14900 epoch=14900/15000 epoch_step=1/1 micro_steps=14900 elapsed=84.6s lr=2.000000e-03 loss=0.1561 loss_recon=0.1561 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9497 corrupt_frac=1.0000 acc_corrupt=0.9497 loss_corrupt=0.1561 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.9088 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6431 out_g_norm=0.1332 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2024 init_gold_top10=0.6107 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1828 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9490 logit_acc_rollout_kept=0.9154 +step=15000 epoch=15000/15000 epoch_step=1/1 micro_steps=15000 elapsed=34.7s lr=2.000000e-03 loss=0.1580 loss_recon=0.1580 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9496 corrupt_frac=1.0000 acc_corrupt=0.9496 loss_corrupt=0.1580 wrong_frac=0.7888 init_acc_corrupt=0.1523 acc_corrupt_t_0p0_0p2=0.9086 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6338 out_g_norm=0.1290 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2158 init_gold_top10=0.5745 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2188 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9366 logit_acc_rollout_kept=0.9387 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=15001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 16000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=15100 epoch=15100/16000 epoch_step=1/1 micro_steps=15100 elapsed=55.5s lr=2.000000e-03 loss=0.1525 loss_recon=0.1525 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9515 corrupt_frac=1.0000 acc_corrupt=0.9515 loss_corrupt=0.1525 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9128 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6246 out_g_norm=0.1324 loss_all=0.0573 init_gold_top10=0.5870 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1949 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9708 logit_acc_rollout_kept=0.9937 +step=15200 epoch=15200/16000 epoch_step=1/1 micro_steps=15200 elapsed=86.8s lr=2.000000e-03 loss=0.1511 loss_recon=0.1511 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9526 corrupt_frac=1.0000 acc_corrupt=0.9526 loss_corrupt=0.1511 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9142 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.6167 out_g_norm=0.1321 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1540 init_gold_top10=0.6027 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1611 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9408 logit_acc_rollout_kept=0.9587 +step=15300 epoch=15300/16000 epoch_step=1/1 micro_steps=15300 elapsed=58.9s lr=2.000000e-03 loss=0.1598 loss_recon=0.1598 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9493 corrupt_frac=1.0000 acc_corrupt=0.9493 loss_corrupt=0.1598 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9101 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6274 out_g_norm=0.1319 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1345 init_gold_top10=0.5549 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1904 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9516 logit_acc_rollout_kept=0.9639 +step=15400 epoch=15400/16000 epoch_step=1/1 micro_steps=15400 elapsed=37.6s lr=2.000000e-03 loss=0.1536 loss_recon=0.1536 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9515 corrupt_frac=1.0000 acc_corrupt=0.9515 loss_corrupt=0.1536 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9129 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6303 out_g_norm=0.1443 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2224 init_gold_top10=0.5154 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1484 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8654 logit_acc_rollout_kept=0.9673 +step=15500 epoch=15500/16000 epoch_step=1/1 micro_steps=15500 elapsed=42.8s lr=2.000000e-03 loss=0.1641 loss_recon=0.1641 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9471 corrupt_frac=1.0000 acc_corrupt=0.9471 loss_corrupt=0.1641 wrong_frac=0.7930 init_acc_corrupt=0.1470 acc_corrupt_t_0p0_0p2=0.9061 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.6293 out_g_norm=0.1321 loss_all=0.1450 init_gold_top10=0.6091 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1779 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9305 logit_acc_rollout_kept=0.9769 +step=15600 epoch=15600/16000 epoch_step=1/1 micro_steps=15600 elapsed=87.9s lr=2.000000e-03 loss=0.1524 loss_recon=0.1524 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9513 corrupt_frac=1.0000 acc_corrupt=0.9513 loss_corrupt=0.1524 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9133 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.6150 out_g_norm=0.1373 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2052 init_gold_top10=0.5742 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1587 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9076 logit_acc_rollout_kept=0.9624 +step=15700 epoch=15700/16000 epoch_step=1/1 micro_steps=15700 elapsed=70.8s lr=2.000000e-03 loss=0.1539 loss_recon=0.1539 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9511 corrupt_frac=1.0000 acc_corrupt=0.9511 loss_corrupt=0.1539 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9119 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6126 out_g_norm=0.1342 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1268 init_gold_top10=0.5896 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1919 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9675 logit_acc_rollout_kept=0.9687 +step=15800 epoch=15800/16000 epoch_step=1/1 micro_steps=15800 elapsed=68.0s lr=2.000000e-03 loss=0.1470 loss_recon=0.1470 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9537 corrupt_frac=1.0000 acc_corrupt=0.9537 loss_corrupt=0.1470 wrong_frac=0.7901 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9165 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6051 out_g_norm=0.1348 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2365 init_gold_top10=0.6102 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2104 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.8852 logit_acc_rollout_kept=0.9772 +step=15900 epoch=15900/16000 epoch_step=1/1 micro_steps=15900 elapsed=87.6s lr=2.000000e-03 loss=0.1521 loss_recon=0.1521 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9516 corrupt_frac=1.0000 acc_corrupt=0.9516 loss_corrupt=0.1521 wrong_frac=0.7895 init_acc_corrupt=0.1515 acc_corrupt_t_0p0_0p2=0.9123 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6075 out_g_norm=0.1363 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1898 init_gold_top10=0.6091 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1827 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.8991 logit_acc_rollout_kept=0.9677 +step=16000 epoch=16000/16000 epoch_step=1/1 micro_steps=16000 elapsed=79.5s lr=2.000000e-03 loss=0.1535 loss_recon=0.1535 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9508 corrupt_frac=1.0000 acc_corrupt=0.9508 loss_corrupt=0.1535 wrong_frac=0.7888 init_acc_corrupt=0.1523 acc_corrupt_t_0p0_0p2=0.9109 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6031 out_g_norm=0.1265 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=0.9996 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2243 init_gold_top10=0.5751 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2191 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9277 logit_acc_rollout_kept=0.9441 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=16001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 17000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=16100 epoch=16100/17000 epoch_step=1/1 micro_steps=16100 elapsed=80.2s lr=2.000000e-03 loss=0.1560 loss_recon=0.1560 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9508 corrupt_frac=1.0000 acc_corrupt=0.9508 loss_corrupt=0.1560 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9115 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6021 out_g_norm=0.1326 loss_all=0.0429 init_gold_top10=0.5885 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1953 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9925 logit_acc_rollout_kept=0.9845 +step=16200 epoch=16200/17000 epoch_step=1/1 micro_steps=16200 elapsed=86.1s lr=2.000000e-03 loss=0.1437 loss_recon=0.1437 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9548 corrupt_frac=1.0000 acc_corrupt=0.9548 loss_corrupt=0.1437 wrong_frac=0.7893 init_acc_corrupt=0.1510 acc_corrupt_t_0p0_0p2=0.9180 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.5993 out_g_norm=0.1329 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1316 init_gold_top10=0.6063 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1614 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9434 logit_acc_rollout_kept=0.9857 +step=16300 epoch=16300/17000 epoch_step=1/1 micro_steps=16300 elapsed=80.1s lr=2.000000e-03 loss=0.1489 loss_recon=0.1489 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9528 corrupt_frac=1.0000 acc_corrupt=0.9528 loss_corrupt=0.1489 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9164 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.6126 out_g_norm=0.1335 acc_corrupt_t_0p8_1p0=0.9993 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1451 init_gold_top10=0.5579 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1906 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9527 logit_acc_rollout_kept=0.9508 +step=16400 epoch=16400/17000 epoch_step=1/1 micro_steps=16400 elapsed=35.3s lr=2.000000e-03 loss=0.1589 loss_recon=0.1589 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9515 corrupt_frac=1.0000 acc_corrupt=0.9515 loss_corrupt=0.1589 wrong_frac=0.7914 init_acc_corrupt=0.1492 acc_corrupt_t_0p0_0p2=0.9129 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.6159 out_g_norm=0.1330 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2840 init_gold_top10=0.5219 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1476 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8665 logit_acc_rollout_kept=0.9332 +step=16500 epoch=16500/17000 epoch_step=1/1 micro_steps=16500 elapsed=39.7s lr=2.000000e-03 loss=0.1586 loss_recon=0.1586 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9489 corrupt_frac=1.0000 acc_corrupt=0.9489 loss_corrupt=0.1586 wrong_frac=0.7930 init_acc_corrupt=0.1470 acc_corrupt_t_0p0_0p2=0.9093 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.6123 out_g_norm=0.1462 loss_all=0.1503 init_gold_top10=0.6057 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1776 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9110 logit_acc_rollout_kept=0.9860 +step=16600 epoch=16600/17000 epoch_step=1/1 micro_steps=16600 elapsed=34.4s lr=2.000000e-03 loss=0.1509 loss_recon=0.1509 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9517 corrupt_frac=1.0000 acc_corrupt=0.9517 loss_corrupt=0.1509 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9140 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.6088 out_g_norm=0.1408 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1747 init_gold_top10=0.5857 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9050 logit_acc_rollout_kept=0.9625 +step=16700 epoch=16700/17000 epoch_step=1/1 micro_steps=16700 elapsed=34.4s lr=2.000000e-03 loss=0.1527 loss_recon=0.1527 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9513 corrupt_frac=1.0000 acc_corrupt=0.9513 loss_corrupt=0.1527 wrong_frac=0.7907 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9122 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.6028 out_g_norm=0.1303 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1375 init_gold_top10=0.5908 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1917 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9623 logit_acc_rollout_kept=0.9537 +step=16800 epoch=16800/17000 epoch_step=1/1 micro_steps=16800 elapsed=34.4s lr=2.000000e-03 loss=0.1486 loss_recon=0.1486 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9534 corrupt_frac=1.0000 acc_corrupt=0.9534 loss_corrupt=0.1486 wrong_frac=0.7901 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9159 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.6144 out_g_norm=0.1348 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1456 init_gold_top10=0.6215 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2106 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9456 logit_acc_rollout_kept=0.9853 +step=16900 epoch=16900/17000 epoch_step=1/1 micro_steps=16900 elapsed=34.4s lr=2.000000e-03 loss=0.1538 loss_recon=0.1538 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9508 corrupt_frac=1.0000 acc_corrupt=0.9508 loss_corrupt=0.1538 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.9110 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=0.9999 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.6142 out_g_norm=0.1361 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1903 init_gold_top10=0.6036 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1830 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9385 logit_acc_rollout_kept=0.9457 +step=17000 epoch=17000/17000 epoch_step=1/1 micro_steps=17000 elapsed=34.4s lr=2.000000e-03 loss=0.1517 loss_recon=0.1517 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9512 corrupt_frac=1.0000 acc_corrupt=0.9512 loss_corrupt=0.1517 wrong_frac=0.7888 init_acc_corrupt=0.1524 acc_corrupt_t_0p0_0p2=0.9115 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.6167 out_g_norm=0.1271 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2396 init_gold_top10=0.5711 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2194 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9356 logit_acc_rollout_kept=0.9314 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=17001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 18000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=17100 epoch=17100/18000 epoch_step=1/1 micro_steps=17100 elapsed=35.1s lr=2.000000e-03 loss=0.1491 loss_recon=0.1491 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9521 corrupt_frac=1.0000 acc_corrupt=0.9521 loss_corrupt=0.1491 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9139 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5963 out_g_norm=0.1222 loss_all=0.0712 init_gold_top10=0.5854 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1952 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9849 logit_acc_rollout_kept=0.9794 +step=17200 epoch=17200/18000 epoch_step=1/1 micro_steps=17200 elapsed=34.4s lr=2.000000e-03 loss=0.1440 loss_recon=0.1440 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9550 corrupt_frac=1.0000 acc_corrupt=0.9550 loss_corrupt=0.1440 wrong_frac=0.7893 init_acc_corrupt=0.1510 acc_corrupt_t_0p0_0p2=0.9184 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.6019 out_g_norm=0.1230 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1854 init_gold_top10=0.6043 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1603 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9389 logit_acc_rollout_kept=0.9705 +step=17300 epoch=17300/18000 epoch_step=1/1 micro_steps=17300 elapsed=34.4s lr=2.000000e-03 loss=0.1516 loss_recon=0.1516 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9522 corrupt_frac=1.0000 acc_corrupt=0.9522 loss_corrupt=0.1516 wrong_frac=0.7934 init_acc_corrupt=0.1465 acc_corrupt_t_0p0_0p2=0.9152 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5937 out_g_norm=0.1349 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1802 init_gold_top10=0.5579 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1909 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9577 logit_acc_rollout_kept=0.9414 +step=17400 epoch=17400/18000 epoch_step=1/1 micro_steps=17400 elapsed=34.4s lr=2.000000e-03 loss=0.1507 loss_recon=0.1507 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9514 corrupt_frac=1.0000 acc_corrupt=0.9514 loss_corrupt=0.1507 wrong_frac=0.7914 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9127 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.5844 out_g_norm=0.1322 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2153 init_gold_top10=0.5273 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1481 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8838 logit_acc_rollout_kept=0.9624 +step=17500 epoch=17500/18000 epoch_step=1/1 micro_steps=17500 elapsed=34.5s lr=2.000000e-03 loss=0.1510 loss_recon=0.1510 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9516 corrupt_frac=1.0000 acc_corrupt=0.9516 loss_corrupt=0.1510 wrong_frac=0.7930 init_acc_corrupt=0.1470 acc_corrupt_t_0p0_0p2=0.9140 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.5850 out_g_norm=0.1270 loss_all=0.0922 init_gold_top10=0.6101 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1778 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9472 logit_acc_rollout_kept=0.9881 +step=17600 epoch=17600/18000 epoch_step=1/1 micro_steps=17600 elapsed=34.4s lr=2.000000e-03 loss=0.1477 loss_recon=0.1477 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9528 corrupt_frac=1.0000 acc_corrupt=0.9528 loss_corrupt=0.1477 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9161 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.5964 out_g_norm=0.1243 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1620 init_gold_top10=0.5758 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1588 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9164 logit_acc_rollout_kept=0.9707 +step=17700 epoch=17700/18000 epoch_step=1/1 micro_steps=17700 elapsed=34.4s lr=2.000000e-03 loss=0.1479 loss_recon=0.1479 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9522 corrupt_frac=1.0000 acc_corrupt=0.9522 loss_corrupt=0.1479 wrong_frac=0.7907 init_acc_corrupt=0.1499 acc_corrupt_t_0p0_0p2=0.9139 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.5870 out_g_norm=0.1303 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1315 init_gold_top10=0.5942 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1921 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9551 logit_acc_rollout_kept=0.9626 +step=17800 epoch=17800/18000 epoch_step=1/1 micro_steps=17800 elapsed=34.4s lr=2.000000e-03 loss=0.1508 loss_recon=0.1508 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9522 corrupt_frac=1.0000 acc_corrupt=0.9522 loss_corrupt=0.1508 wrong_frac=0.7901 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9138 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.5789 out_g_norm=0.1301 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=0.9990 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2459 init_gold_top10=0.6180 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2103 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.8948 logit_acc_rollout_kept=0.9566 +step=17900 epoch=17900/18000 epoch_step=1/1 micro_steps=17900 elapsed=34.5s lr=2.000000e-03 loss=0.1481 loss_recon=0.1481 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9522 corrupt_frac=1.0000 acc_corrupt=0.9522 loss_corrupt=0.1481 wrong_frac=0.7895 init_acc_corrupt=0.1515 acc_corrupt_t_0p0_0p2=0.9134 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.5779 out_g_norm=0.1225 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1715 init_gold_top10=0.6092 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1831 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9325 logit_acc_rollout_kept=0.9486 +step=18000 epoch=18000/18000 epoch_step=1/1 micro_steps=18000 elapsed=34.4s lr=2.000000e-03 loss=0.1515 loss_recon=0.1515 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9521 corrupt_frac=1.0000 acc_corrupt=0.9521 loss_corrupt=0.1515 wrong_frac=0.7888 init_acc_corrupt=0.1523 acc_corrupt_t_0p0_0p2=0.9132 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.5655 out_g_norm=0.1217 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1335 init_gold_top10=0.5755 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2198 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9679 logit_acc_rollout_kept=0.9480 +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=18001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 19000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=18100 epoch=18100/19000 epoch_step=1/1 micro_steps=18100 elapsed=35.1s lr=2.000000e-03 loss=0.1478 loss_recon=0.1478 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9528 corrupt_frac=1.0000 acc_corrupt=0.9528 loss_corrupt=0.1478 wrong_frac=0.7918 init_acc_corrupt=0.1491 acc_corrupt_t_0p0_0p2=0.9151 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5416 out_g_norm=0.1279 loss_all=0.0867 init_gold_top10=0.5859 init_gold_top100=0.6537 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1948 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9810 logit_acc_rollout_kept=0.9622 +step=18200 epoch=18200/19000 epoch_step=1/1 micro_steps=18200 elapsed=34.4s lr=2.000000e-03 loss=0.1494 loss_recon=0.1494 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9526 corrupt_frac=1.0000 acc_corrupt=0.9526 loss_corrupt=0.1494 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9141 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.5371 out_g_norm=0.1204 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1614 init_gold_top10=0.6047 init_gold_top100=0.6795 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1611 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9168 logit_acc_rollout_kept=0.9799 +step=18300 epoch=18300/19000 epoch_step=1/1 micro_steps=18300 elapsed=34.4s lr=2.000000e-03 loss=0.1524 loss_recon=0.1524 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9513 corrupt_frac=1.0000 acc_corrupt=0.9513 loss_corrupt=0.1524 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9138 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5397 out_g_norm=0.1285 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1838 init_gold_top10=0.5540 init_gold_top100=0.6413 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1901 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9372 logit_acc_rollout_kept=0.9494 +step=18400 epoch=18400/19000 epoch_step=1/1 micro_steps=18400 elapsed=34.4s lr=2.000000e-03 loss=0.1457 loss_recon=0.1457 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9542 corrupt_frac=1.0000 acc_corrupt=0.9542 loss_corrupt=0.1457 wrong_frac=0.7914 init_acc_corrupt=0.1492 acc_corrupt_t_0p0_0p2=0.9176 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.5441 out_g_norm=0.1301 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1665 init_gold_top10=0.5256 init_gold_top100=0.6166 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1481 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.9144 logit_acc_rollout_kept=0.9704 +step=18500 epoch=18500/19000 epoch_step=1/1 micro_steps=18500 elapsed=34.4s lr=2.000000e-03 loss=0.1490 loss_recon=0.1490 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9528 corrupt_frac=1.0000 acc_corrupt=0.9528 loss_corrupt=0.1490 wrong_frac=0.7930 init_acc_corrupt=0.1470 acc_corrupt_t_0p0_0p2=0.9162 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.5344 out_g_norm=0.1271 loss_all=0.1282 init_gold_top10=0.6026 init_gold_top100=0.6804 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1777 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9437 logit_acc_rollout_kept=0.9833 +step=18600 epoch=18600/19000 epoch_step=1/1 micro_steps=18600 elapsed=34.4s lr=2.000000e-03 loss=0.1480 loss_recon=0.1480 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9532 corrupt_frac=1.0000 acc_corrupt=0.9532 loss_corrupt=0.1480 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9166 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.5280 out_g_norm=0.1274 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2018 init_gold_top10=0.5753 init_gold_top100=0.6594 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.8978 logit_acc_rollout_kept=0.9656 +step=18700 epoch=18700/19000 epoch_step=1/1 micro_steps=18700 elapsed=34.4s lr=2.000000e-03 loss=0.1459 loss_recon=0.1459 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9538 corrupt_frac=1.0000 acc_corrupt=0.9538 loss_corrupt=0.1459 wrong_frac=0.7907 init_acc_corrupt=0.1499 acc_corrupt_t_0p0_0p2=0.9167 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.5263 out_g_norm=0.1208 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0710 init_gold_top10=0.5959 init_gold_top100=0.6663 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1919 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9800 logit_acc_rollout_kept=0.9733 +step=18800 epoch=18800/19000 epoch_step=1/1 micro_steps=18800 elapsed=34.4s lr=2.000000e-03 loss=0.1454 loss_recon=0.1454 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9535 corrupt_frac=1.0000 acc_corrupt=0.9535 loss_corrupt=0.1454 wrong_frac=0.7901 init_acc_corrupt=0.1499 acc_corrupt_t_0p0_0p2=0.9161 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.5331 out_g_norm=0.1209 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1459 init_gold_top10=0.6239 init_gold_top100=0.6948 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2100 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9313 logit_acc_rollout_kept=0.9797 +step=18900 epoch=18900/19000 epoch_step=1/1 micro_steps=18900 elapsed=34.4s lr=2.000000e-03 loss=0.1469 loss_recon=0.1469 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9532 corrupt_frac=1.0000 acc_corrupt=0.9532 loss_corrupt=0.1469 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.9153 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.5210 out_g_norm=0.1247 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1085 init_gold_top10=0.6124 init_gold_top100=0.6777 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1830 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9560 logit_acc_rollout_kept=0.9742 +step=19000 epoch=19000/19000 epoch_step=1/1 micro_steps=19000 elapsed=34.4s lr=2.000000e-03 loss=0.1466 loss_recon=0.1466 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9529 corrupt_frac=1.0000 acc_corrupt=0.9529 loss_corrupt=0.1466 wrong_frac=0.7888 init_acc_corrupt=0.1523 acc_corrupt_t_0p0_0p2=0.9146 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.5219 out_g_norm=0.1224 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1617 init_gold_top10=0.5773 init_gold_top100=0.6548 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2194 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9485 logit_acc_rollout_kept=0.9743 +scripts/launch_lta_owt_gpt2_softendpoint_mn_pilot_4gpu.sh: line 254: istic_normal_tau_min: command not found +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt start_step=19001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2664, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 20000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2616320, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_infer_steps": 32, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_rollin_len1024_rollin_p50_s4_i32_20260517_1855ctx1024bs128/latest.pt" +} +step=19100 epoch=19100/20000 epoch_step=1/1 micro_steps=19100 elapsed=24.7s lr=2.000000e-03 loss=0.1478 loss_recon=0.1478 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9526 corrupt_frac=1.0000 acc_corrupt=0.9526 loss_corrupt=0.1478 wrong_frac=0.7918 init_acc_corrupt=0.1492 acc_corrupt_t_0p0_0p2=0.9147 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5213 out_g_norm=0.1252 loss_all=0.0607 init_gold_top10=0.5851 init_gold_top100=0.6517 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1945 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.9675 logit_acc_rollout_kept=0.9920 +step=19200 epoch=19200/20000 epoch_step=1/1 micro_steps=19200 elapsed=23.5s lr=2.000000e-03 loss=0.1439 loss_recon=0.1439 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9544 corrupt_frac=1.0000 acc_corrupt=0.9544 loss_corrupt=0.1439 wrong_frac=0.7893 init_acc_corrupt=0.1509 acc_corrupt_t_0p0_0p2=0.9174 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=12.5133 out_g_norm=0.1299 acc_corrupt_t_0p8_1p0=0.9997 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1535 init_gold_top10=0.6043 init_gold_top100=0.6775 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1601 init_acc_rollout_kept=0.1259 logit_acc_rollout_applied=0.9212 logit_acc_rollout_kept=0.9848 +step=19300 epoch=19300/20000 epoch_step=1/1 micro_steps=19300 elapsed=23.6s lr=2.000000e-03 loss=0.1494 loss_recon=0.1494 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9527 corrupt_frac=1.0000 acc_corrupt=0.9527 loss_corrupt=0.1494 wrong_frac=0.7934 init_acc_corrupt=0.1464 acc_corrupt_t_0p0_0p2=0.9162 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=12.5157 out_g_norm=0.1237 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1804 init_gold_top10=0.5556 init_gold_top100=0.6388 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1906 init_acc_rollout_kept=0.1130 logit_acc_rollout_applied=0.9467 logit_acc_rollout_kept=0.9466 +step=19400 epoch=19400/20000 epoch_step=1/1 micro_steps=19400 elapsed=23.8s lr=2.000000e-03 loss=0.1542 loss_recon=0.1542 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9512 corrupt_frac=1.0000 acc_corrupt=0.9512 loss_corrupt=0.1542 wrong_frac=0.7914 init_acc_corrupt=0.1492 acc_corrupt_t_0p0_0p2=0.9123 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=12.5228 out_g_norm=0.1234 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2543 init_gold_top10=0.5313 init_gold_top100=0.6136 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.1478 init_acc_rollout_kept=0.1081 logit_acc_rollout_applied=0.8905 logit_acc_rollout_kept=0.9370 +step=19500 epoch=19500/20000 epoch_step=1/1 micro_steps=19500 elapsed=23.8s lr=2.000000e-03 loss=0.1441 loss_recon=0.1441 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9543 corrupt_frac=1.0000 acc_corrupt=0.9543 loss_corrupt=0.1441 wrong_frac=0.7930 init_acc_corrupt=0.1470 acc_corrupt_t_0p0_0p2=0.9189 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=12.5182 out_g_norm=0.1287 loss_all=0.1263 init_gold_top10=0.6042 init_gold_top100=0.6781 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1780 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.9236 logit_acc_rollout_kept=0.9846 +step=19600 epoch=19600/20000 epoch_step=1/1 micro_steps=19600 elapsed=23.8s lr=2.000000e-03 loss=0.1470 loss_recon=0.1470 loss_meanflow=0.0000 mean_model_t=0.2074 mean_corrupt_t=0.2074 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9540 corrupt_frac=1.0000 acc_corrupt=0.9540 loss_corrupt=0.1470 wrong_frac=0.7925 init_acc_corrupt=0.1468 acc_corrupt_t_0p0_0p2=0.9181 corrupt_frac_t_0p0_0p2=0.5617 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0729 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0124 out_w_norm=12.5085 out_g_norm=0.1246 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2038 init_gold_top10=0.5801 init_gold_top100=0.6569 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1586 init_acc_rollout_kept=0.1085 logit_acc_rollout_applied=0.9001 logit_acc_rollout_kept=0.9622 +step=19700 epoch=19700/20000 epoch_step=1/1 micro_steps=19700 elapsed=23.7s lr=2.000000e-03 loss=0.1448 loss_recon=0.1448 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5012 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9546 corrupt_frac=1.0000 acc_corrupt=0.9546 loss_corrupt=0.1448 wrong_frac=0.7907 init_acc_corrupt=0.1499 acc_corrupt_t_0p0_0p2=0.9183 corrupt_frac_t_0p0_0p2=0.5550 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3587 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=12.4992 out_g_norm=0.1299 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.0743 init_gold_top10=0.5995 init_gold_top100=0.6638 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.1915 init_acc_rollout_kept=0.1291 logit_acc_rollout_applied=0.9811 logit_acc_rollout_kept=0.9715 +step=19800 epoch=19800/20000 epoch_step=1/1 micro_steps=19800 elapsed=23.6s lr=2.000000e-03 loss=0.1403 loss_recon=0.1403 loss_meanflow=0.0000 mean_model_t=0.2099 mean_corrupt_t=0.2099 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4994 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9556 corrupt_frac=1.0000 acc_corrupt=0.9556 loss_corrupt=0.1403 wrong_frac=0.7901 init_acc_corrupt=0.1498 acc_corrupt_t_0p0_0p2=0.9200 corrupt_frac_t_0p0_0p2=0.5548 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3585 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0790 out_w_norm=12.4940 out_g_norm=0.1279 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.0130 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.2023 init_gold_top10=0.6174 init_gold_top100=0.6929 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.2109 init_acc_rollout_kept=0.1370 logit_acc_rollout_applied=0.9194 logit_acc_rollout_kept=0.9831 +step=19900 epoch=19900/20000 epoch_step=1/1 micro_steps=19900 elapsed=23.6s lr=2.000000e-03 loss=0.1448 loss_recon=0.1448 loss_meanflow=0.0000 mean_model_t=0.2107 mean_corrupt_t=0.2107 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4979 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9543 corrupt_frac=1.0000 acc_corrupt=0.9543 loss_corrupt=0.1448 wrong_frac=0.7895 init_acc_corrupt=0.1514 acc_corrupt_t_0p0_0p2=0.9173 corrupt_frac_t_0p0_0p2=0.5521 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3599 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0788 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0133 out_w_norm=12.4878 out_g_norm=0.1325 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1480 init_gold_top10=0.6096 init_gold_top100=0.6753 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.1829 init_acc_rollout_kept=0.1128 logit_acc_rollout_applied=0.9526 logit_acc_rollout_kept=0.9387 +step=20000 epoch=20000/20000 epoch_step=1/1 micro_steps=20000 elapsed=23.8s lr=2.000000e-03 loss=0.1459 loss_recon=0.1459 loss_meanflow=0.0000 mean_model_t=0.2112 mean_corrupt_t=0.2112 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4985 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9531 corrupt_frac=1.0000 acc_corrupt=0.9531 loss_corrupt=0.1459 wrong_frac=0.7888 init_acc_corrupt=0.1522 acc_corrupt_t_0p0_0p2=0.9150 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=1.0000 corrupt_frac_t_0p2_0p4=0.3570 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.0810 out_w_norm=12.4866 out_g_norm=0.1279 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.0137 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.1319 init_gold_top10=0.5827 init_gold_top100=0.6513 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.2178 init_acc_rollout_kept=0.1012 logit_acc_rollout_applied=0.9369 logit_acc_rollout_kept=0.9674 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_synct_len256_synct_p50_s8_i64_20260517_1800synct.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_synct_len256_synct_p50_s8_i64_20260517_1800synct.log new file mode 100644 index 0000000000000000000000000000000000000000..88b5106e58ec9de35e82ca6f3dac784c8e4ee5bb --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_rollin_synct_len256_synct_p50_s8_i64_20260517_1800synct.log @@ -0,0 +1,224 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_rollin_synct_len256_synct_p50_s8_i64_20260517_1800synct", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 500, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 8, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 500, + "resume_path": "" +} +step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=11.2s lr=2.000000e-03 loss=6.7066 loss_recon=6.7066 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0995 corrupt_frac=1.0000 acc_corrupt=0.0995 loss_corrupt=6.7066 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0486 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1326 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2811 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.4045 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0999 out_g_norm=1.0064 loss_all=6.4488 init_gold_top10=0.2091 init_gold_top100=0.4887 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1132 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.1071 logit_acc_rollout_kept=0.0991 +W0517 17:58:23.621000 251950 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0517 17:58:23.623000 251950 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 251955 closing signal SIGTERM +W0517 17:58:23.624000 251950 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 251956 closing signal SIGTERM +W0517 17:58:23.624000 251950 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 251957 closing signal SIGTERM +W0517 17:58:23.625000 251950 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 251958 closing signal SIGTERM +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 251950 got signal: 15 diff --git a/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_wrongfloor_len256_wrongfloor0p3_20260517_1815wrongfloor.log b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_wrongfloor_len256_wrongfloor0p3_20260517_1815wrongfloor.log new file mode 100644 index 0000000000000000000000000000000000000000..833c79c581ac99862978345ac10ba0b2bf1cbae2 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_wrongfloor_len256_wrongfloor0p3_20260517_1815wrongfloor.log @@ -0,0 +1,199 @@ +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 969, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/train8_wrongfloor_len256_wrongfloor0p3_20260517_1815wrongfloor", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 1965440, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "logit_normal", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.3, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "rollout_train_sync_t": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=4.4s lr=2.000000e-03 loss=6.7048 loss_recon=6.7048 loss_meanflow=0.0000 mean_model_t=0.2082 mean_corrupt_t=0.2082 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0998 corrupt_frac=1.0000 acc_corrupt=0.0998 loss_corrupt=6.7048 wrong_frac=0.7918 init_acc_corrupt=0.1152 acc_corrupt_t_0p0_0p2=0.0485 corrupt_frac_t_0p0_0p2=0.5588 acc_corrupt_t_0p2_0p4=0.1343 corrupt_frac_t_0p2_0p4=0.3579 acc_corrupt_t_0p4_0p6=0.2822 corrupt_frac_t_0p4_0p6=0.0752 acc_corrupt_t_0p6_0p8=0.4246 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=1.0992 out_g_norm=1.0008 acc_corrupt_t_0p8_1p0=0.4414 corrupt_frac_t_0p8_1p0=0.0078 loss_all=6.4736 init_gold_top10=0.1887 init_gold_top100=0.4165 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=3.7s lr=2.000000e-03 loss=6.1011 loss_recon=6.1011 loss_meanflow=0.0000 mean_model_t=0.2081 mean_corrupt_t=0.2081 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1091 corrupt_frac=1.0000 acc_corrupt=0.1091 loss_corrupt=6.1011 wrong_frac=0.7926 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.0546 corrupt_frac_t_0p0_0p2=0.5577 acc_corrupt_t_0p2_0p4=0.1490 corrupt_frac_t_0p2_0p4=0.3608 acc_corrupt_t_0p4_0p6=0.2945 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4062 corrupt_frac_t_0p6_0p8=0.0119 out_w_norm=3.3219 out_g_norm=1.4082 acc_corrupt_t_0p8_1p0=0.4609 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8429 init_gold_top10=0.1850 init_gold_top100=0.4143 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=3.7s lr=2.000000e-03 loss=5.5386 loss_recon=5.5386 loss_meanflow=0.0000 mean_model_t=0.2117 mean_corrupt_t=0.2117 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1243 corrupt_frac=1.0000 acc_corrupt=0.1243 loss_corrupt=5.5386 wrong_frac=0.7885 init_acc_corrupt=0.1197 acc_corrupt_t_0p0_0p2=0.0591 corrupt_frac_t_0p0_0p2=0.5484 acc_corrupt_t_0p2_0p4=0.1681 corrupt_frac_t_0p2_0p4=0.3611 acc_corrupt_t_0p4_0p6=0.3285 corrupt_frac_t_0p4_0p6=0.0803 acc_corrupt_t_0p6_0p8=0.4765 corrupt_frac_t_0p6_0p8=0.0139 out_w_norm=5.2102 out_g_norm=0.7079 acc_corrupt_t_0p8_1p0=0.4980 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.2636 init_gold_top10=0.2059 init_gold_top100=0.4209 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=3.7s lr=2.000000e-03 loss=5.0143 loss_recon=5.0143 loss_meanflow=0.0000 mean_model_t=0.2073 mean_corrupt_t=0.2073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1462 corrupt_frac=1.0000 acc_corrupt=0.1462 loss_corrupt=5.0143 wrong_frac=0.7928 init_acc_corrupt=0.1147 acc_corrupt_t_0p0_0p2=0.0638 corrupt_frac_t_0p0_0p2=0.5613 acc_corrupt_t_0p2_0p4=0.2018 corrupt_frac_t_0p2_0p4=0.3571 acc_corrupt_t_0p4_0p6=0.4463 corrupt_frac_t_0p4_0p6=0.0727 acc_corrupt_t_0p6_0p8=0.6573 corrupt_frac_t_0p6_0p8=0.0126 out_w_norm=6.8955 out_g_norm=0.4054 loss_all=4.6717 init_gold_top10=0.2018 init_gold_top100=0.4261 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=3.7s lr=2.000000e-03 loss=4.2633 loss_recon=4.2633 loss_meanflow=0.0000 mean_model_t=0.2097 mean_corrupt_t=0.2097 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1848 corrupt_frac=1.0000 acc_corrupt=0.1848 loss_corrupt=4.2633 wrong_frac=0.7905 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.0726 corrupt_frac_t_0p0_0p2=0.5540 acc_corrupt_t_0p2_0p4=0.2715 corrupt_frac_t_0p2_0p4=0.3605 acc_corrupt_t_0p4_0p6=0.5261 corrupt_frac_t_0p4_0p6=0.0745 out_w_norm=8.4270 out_g_norm=0.4608 acc_corrupt_t_0p6_0p8=0.6848 corrupt_frac_t_0p6_0p8=0.0139 acc_corrupt_t_0p8_1p0=0.7448 corrupt_frac_t_0p8_1p0=0.0078 loss_all=3.8613 init_gold_top10=0.2070 init_gold_top100=0.4169 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=3.7s lr=2.000000e-03 loss=3.4251 loss_recon=3.4251 loss_meanflow=0.0000 mean_model_t=0.2089 mean_corrupt_t=0.2089 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2389 corrupt_frac=1.0000 acc_corrupt=0.2389 loss_corrupt=3.4251 wrong_frac=0.7911 init_acc_corrupt=0.1167 acc_corrupt_t_0p0_0p2=0.0945 corrupt_frac_t_0p0_0p2=0.5598 acc_corrupt_t_0p2_0p4=0.3712 corrupt_frac_t_0p2_0p4=0.3568 acc_corrupt_t_0p4_0p6=0.6290 corrupt_frac_t_0p4_0p6=0.0750 acc_corrupt_t_0p6_0p8=0.7620 corrupt_frac_t_0p6_0p8=0.0120 out_w_norm=9.5765 out_g_norm=0.5181 acc_corrupt_t_0p8_1p0=0.7689 corrupt_frac_t_0p8_1p0=0.0094 loss_all=3.1182 init_gold_top10=0.1900 init_gold_top100=0.4196 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=3.7s lr=2.000000e-03 loss=2.6250 loss_recon=2.6250 loss_meanflow=0.0000 mean_model_t=0.2095 mean_corrupt_t=0.2095 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3370 corrupt_frac=1.0000 acc_corrupt=0.3370 loss_corrupt=2.6250 wrong_frac=0.7907 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.1399 corrupt_frac_t_0p0_0p2=0.5552 acc_corrupt_t_0p2_0p4=0.5298 corrupt_frac_t_0p2_0p4=0.3584 acc_corrupt_t_0p4_0p6=0.7932 corrupt_frac_t_0p4_0p6=0.0764 out_w_norm=10.2365 out_g_norm=0.6321 acc_corrupt_t_0p6_0p8=0.8852 corrupt_frac_t_0p6_0p8=0.0132 acc_corrupt_t_0p8_1p0=0.8451 corrupt_frac_t_0p8_1p0=0.0078 loss_all=2.2547 init_gold_top10=0.1937 init_gold_top100=0.4183 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=3.7s lr=2.000000e-03 loss=1.8240 loss_recon=1.8240 loss_meanflow=0.0000 mean_model_t=0.2103 mean_corrupt_t=0.2103 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4941 corrupt_frac=1.0000 acc_corrupt=0.4941 loss_corrupt=1.8240 wrong_frac=0.7898 init_acc_corrupt=0.1186 acc_corrupt_t_0p0_0p2=0.2539 corrupt_frac_t_0p0_0p2=0.5525 acc_corrupt_t_0p2_0p4=0.7542 corrupt_frac_t_0p2_0p4=0.3592 acc_corrupt_t_0p4_0p6=0.9359 corrupt_frac_t_0p4_0p6=0.0800 acc_corrupt_t_0p6_0p8=0.9643 corrupt_frac_t_0p6_0p8=0.0141 out_w_norm=10.6284 out_g_norm=0.8746 acc_corrupt_t_0p8_1p0=0.9531 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.4551 init_gold_top10=0.2038 init_gold_top100=0.4186 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=3.7s lr=2.000000e-03 loss=1.1915 loss_recon=1.1915 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6347 corrupt_frac=1.0000 acc_corrupt=0.6347 loss_corrupt=1.1915 wrong_frac=0.7907 init_acc_corrupt=0.1180 acc_corrupt_t_0p0_0p2=0.3964 corrupt_frac_t_0p0_0p2=0.5584 acc_corrupt_t_0p2_0p4=0.9225 corrupt_frac_t_0p2_0p4=0.3530 acc_corrupt_t_0p4_0p6=0.9890 corrupt_frac_t_0p4_0p6=0.0801 acc_corrupt_t_0p6_0p8=0.9912 corrupt_frac_t_0p6_0p8=0.0125 out_w_norm=10.9390 out_g_norm=0.9790 acc_corrupt_t_0p8_1p0=0.9727 corrupt_frac_t_0p8_1p0=0.0078 loss_all=1.0933 init_gold_top10=0.2031 init_gold_top100=0.4228 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=3.7s lr=2.000000e-03 loss=0.8912 loss_recon=0.8912 loss_meanflow=0.0000 mean_model_t=0.2094 mean_corrupt_t=0.2094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7007 corrupt_frac=1.0000 acc_corrupt=0.7007 loss_corrupt=0.8912 wrong_frac=0.7908 init_acc_corrupt=0.1167 acc_corrupt_t_0p0_0p2=0.4763 corrupt_frac_t_0p0_0p2=0.5561 acc_corrupt_t_0p2_0p4=0.9778 corrupt_frac_t_0p2_0p4=0.3595 acc_corrupt_t_0p4_0p6=0.9982 corrupt_frac_t_0p4_0p6=0.0759 acc_corrupt_t_0p6_0p8=0.9958 corrupt_frac_t_0p6_0p8=0.0129 out_w_norm=11.1022 out_g_norm=0.9477 acc_corrupt_t_0p8_1p0=0.9805 corrupt_frac_t_0p8_1p0=0.0078 loss_all=0.8570 init_gold_top10=0.2021 init_gold_top100=0.4184 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/RECORD b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..97ceb4a0b85a55de8cad92c06976747a22df38ff --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/RECORD @@ -0,0 +1,9 @@ +hf_xet-1.5.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +hf_xet-1.5.0.dist-info/METADATA,sha256=kiRjS5pSbyNKVHLD5pYclp87MWjinAwQrmuPjKJX8yA,4882 +hf_xet-1.5.0.dist-info/RECORD,, +hf_xet-1.5.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +hf_xet-1.5.0.dist-info/WHEEL,sha256=LbLhSGTXlKRmTCKROpRhcqPcmllwkP3mQdJf3GnRbUM,143 +hf_xet-1.5.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 +hf_xet-1.5.0.dist-info/sboms/hf_xet.cyclonedx.json,sha256=cqFELPUEOVkJUBakT3vgAWYE1N939XMjcP_vAKt_0xw,305769 +hf_xet/__init__.py,sha256=E8UDdyQ8glZ_nve9hHEf22bPang8-RKx4VuApXYeQUo,107 +hf_xet/hf_xet.abi3.so,sha256=6s8jp7y5mQ7kVbKGwgTcVFDzZ8rBdC63i7CPjbm_wEk,11465992 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/WHEEL b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..2598d7e7f7c35b338af38cf8eabcd74b0c04b83f --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: maturin (1.13.1) +Root-Is-Purelib: false +Tag: cp37-abi3-manylinux_2_17_x86_64 +Tag: cp37-abi3-manylinux2014_x86_64 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/anyio.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/anyio.py new file mode 100644 index 0000000000000000000000000000000000000000..a140095e1b8de022f321a41c0125e0e5febc0749 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/anyio.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import ssl +import typing + +import anyio + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .._utils import is_socket_readable +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class AnyIOStream(AsyncNetworkStream): + def __init__(self, stream: anyio.abc.ByteStream) -> None: + self._stream = stream + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map = { + TimeoutError: ReadTimeout, + anyio.BrokenResourceError: ReadError, + anyio.ClosedResourceError: ReadError, + anyio.EndOfStream: ReadError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + try: + return await self._stream.receive(max_bytes=max_bytes) + except anyio.EndOfStream: # pragma: nocover + return b"" + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + exc_map = { + TimeoutError: WriteTimeout, + anyio.BrokenResourceError: WriteError, + anyio.ClosedResourceError: WriteError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + await self._stream.send(item=buffer) + + async def aclose(self) -> None: + await self._stream.aclose() + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + exc_map = { + TimeoutError: ConnectTimeout, + anyio.BrokenResourceError: ConnectError, + anyio.EndOfStream: ConnectError, + ssl.SSLError: ConnectError, + } + with map_exceptions(exc_map): + try: + with anyio.fail_after(timeout): + ssl_stream = await anyio.streams.tls.TLSStream.wrap( + self._stream, + ssl_context=ssl_context, + hostname=server_hostname, + standard_compatible=False, + server_side=False, + ) + except Exception as exc: # pragma: nocover + await self.aclose() + raise exc + return AnyIOStream(ssl_stream) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object": + return self._stream.extra(anyio.streams.tls.TLSAttribute.ssl_object, None) + if info == "client_addr": + return self._stream.extra(anyio.abc.SocketAttribute.local_address, None) + if info == "server_addr": + return self._stream.extra(anyio.abc.SocketAttribute.remote_address, None) + if info == "socket": + return self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + if info == "is_readable": + sock = self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + return is_socket_readable(sock) + return None + + +class AnyIOBackend(AsyncNetworkBackend): + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_tcp( + remote_host=host, + remote_port=port, + local_host=local_address, + ) + # By default TCP sockets opened in `asyncio` include TCP_NODELAY. + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_unix(path) + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def sleep(self, seconds: float) -> None: + await anyio.sleep(seconds) # pragma: nocover diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/auto.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/auto.py new file mode 100644 index 0000000000000000000000000000000000000000..49f0e698c97ad5623f376d8182675352e21c2c3c --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/auto.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import typing + +from .._synchronization import current_async_library +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class AutoBackend(AsyncNetworkBackend): + async def _init_backend(self) -> None: + if not (hasattr(self, "_backend")): + backend = current_async_library() + if backend == "trio": + from .trio import TrioBackend + + self._backend: AsyncNetworkBackend = TrioBackend() + else: + from .anyio import AnyIOBackend + + self._backend = AnyIOBackend() + + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + await self._init_backend() + return await self._backend.connect_tcp( + host, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + await self._init_backend() + return await self._backend.connect_unix_socket( + path, timeout=timeout, socket_options=socket_options + ) + + async def sleep(self, seconds: float) -> None: # pragma: nocover + await self._init_backend() + return await self._backend.sleep(seconds) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/base.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..cf55c8b10eb543872550be863206fe2f760d0d8d --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/base.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import ssl +import time +import typing + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + + +class NetworkStream: + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + def close(self) -> None: + raise NotImplementedError() # pragma: nocover + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class NetworkBackend: + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def sleep(self, seconds: float) -> None: + time.sleep(seconds) # pragma: nocover + + +class AsyncNetworkStream: + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + async def aclose(self) -> None: + raise NotImplementedError() # pragma: nocover + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class AsyncNetworkBackend: + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def sleep(self, seconds: float) -> None: + raise NotImplementedError() # pragma: nocover diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/mock.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..9b6edca03d4d4b34f355fd53e49d4b4c699c972c --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/mock.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import ssl +import typing + +from .._exceptions import ReadError +from .base import ( + SOCKET_OPTION, + AsyncNetworkBackend, + AsyncNetworkStream, + NetworkBackend, + NetworkStream, +) + + +class MockSSLObject: + def __init__(self, http2: bool): + self._http2 = http2 + + def selected_alpn_protocol(self) -> str: + return "h2" if self._http2 else "http/1.1" + + +class MockStream(NetworkStream): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + self._closed = False + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + if self._closed: + raise ReadError("Connection closed") + if not self._buffer: + return b"" + return self._buffer.pop(0) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + pass + + def close(self) -> None: + self._closed = True + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + return self + + def get_extra_info(self, info: str) -> typing.Any: + return MockSSLObject(http2=self._http2) if info == "ssl_object" else None + + def __repr__(self) -> str: + return "" + + +class MockBackend(NetworkBackend): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + return MockStream(list(self._buffer), http2=self._http2) + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + return MockStream(list(self._buffer), http2=self._http2) + + def sleep(self, seconds: float) -> None: + pass + + +class AsyncMockStream(AsyncNetworkStream): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + self._closed = False + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + if self._closed: + raise ReadError("Connection closed") + if not self._buffer: + return b"" + return self._buffer.pop(0) + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + pass + + async def aclose(self) -> None: + self._closed = True + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + return self + + def get_extra_info(self, info: str) -> typing.Any: + return MockSSLObject(http2=self._http2) if info == "ssl_object" else None + + def __repr__(self) -> str: + return "" + + +class AsyncMockBackend(AsyncNetworkBackend): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + return AsyncMockStream(list(self._buffer), http2=self._http2) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + return AsyncMockStream(list(self._buffer), http2=self._http2) + + async def sleep(self, seconds: float) -> None: + pass diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/sync.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/sync.py new file mode 100644 index 0000000000000000000000000000000000000000..4018a09c6fb1e0ef1b03ab8d84b13ebef4031f7c --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/sync.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import functools +import socket +import ssl +import sys +import typing + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ExceptionMapping, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .._utils import is_socket_readable +from .base import SOCKET_OPTION, NetworkBackend, NetworkStream + + +class TLSinTLSStream(NetworkStream): # pragma: no cover + """ + Because the standard `SSLContext.wrap_socket` method does + not work for `SSLSocket` objects, we need this class + to implement TLS stream using an underlying `SSLObject` + instance in order to support TLS on top of TLS. + """ + + # Defined in RFC 8449 + TLS_RECORD_SIZE = 16384 + + def __init__( + self, + sock: socket.socket, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ): + self._sock = sock + self._incoming = ssl.MemoryBIO() + self._outgoing = ssl.MemoryBIO() + + self.ssl_obj = ssl_context.wrap_bio( + incoming=self._incoming, + outgoing=self._outgoing, + server_hostname=server_hostname, + ) + + self._sock.settimeout(timeout) + self._perform_io(self.ssl_obj.do_handshake) + + def _perform_io( + self, + func: typing.Callable[..., typing.Any], + ) -> typing.Any: + ret = None + + while True: + errno = None + try: + ret = func() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError) as e: + errno = e.errno + + self._sock.sendall(self._outgoing.read()) + + if errno == ssl.SSL_ERROR_WANT_READ: + buf = self._sock.recv(self.TLS_RECORD_SIZE) + + if buf: + self._incoming.write(buf) + else: + self._incoming.write_eof() + if errno is None: + return ret + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + return typing.cast( + bytes, self._perform_io(functools.partial(self.ssl_obj.read, max_bytes)) + ) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + while buffer: + nsent = self._perform_io(functools.partial(self.ssl_obj.write, buffer)) + buffer = buffer[nsent:] + + def close(self) -> None: + self._sock.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + raise NotImplementedError() + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object": + return self.ssl_obj + if info == "client_addr": + return self._sock.getsockname() + if info == "server_addr": + return self._sock.getpeername() + if info == "socket": + return self._sock + if info == "is_readable": + return is_socket_readable(self._sock) + return None + + +class SyncStream(NetworkStream): + def __init__(self, sock: socket.socket) -> None: + self._sock = sock + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + return self._sock.recv(max_bytes) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} + with map_exceptions(exc_map): + while buffer: + self._sock.settimeout(timeout) + n = self._sock.send(buffer) + buffer = buffer[n:] + + def close(self) -> None: + self._sock.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + with map_exceptions(exc_map): + try: + if isinstance(self._sock, ssl.SSLSocket): # pragma: no cover + # If the underlying socket has already been upgraded + # to the TLS layer (i.e. is an instance of SSLSocket), + # we need some additional smarts to support TLS-in-TLS. + return TLSinTLSStream( + self._sock, ssl_context, server_hostname, timeout + ) + else: + self._sock.settimeout(timeout) + sock = ssl_context.wrap_socket( + self._sock, server_hostname=server_hostname + ) + except Exception as exc: # pragma: nocover + self.close() + raise exc + return SyncStream(sock) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object" and isinstance(self._sock, ssl.SSLSocket): + return self._sock._sslobj # type: ignore + if info == "client_addr": + return self._sock.getsockname() + if info == "server_addr": + return self._sock.getpeername() + if info == "socket": + return self._sock + if info == "is_readable": + return is_socket_readable(self._sock) + return None + + +class SyncBackend(NetworkBackend): + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + # Note that we automatically include `TCP_NODELAY` + # in addition to any other custom socket options. + if socket_options is None: + socket_options = [] # pragma: no cover + address = (host, port) + source_address = None if local_address is None else (local_address, 0) + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + + with map_exceptions(exc_map): + sock = socket.create_connection( + address, + timeout, + source_address=source_address, + ) + for option in socket_options: + sock.setsockopt(*option) # pragma: no cover + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return SyncStream(sock) + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: # pragma: nocover + if sys.platform == "win32": + raise RuntimeError( + "Attempted to connect to a UNIX socket on a Windows system." + ) + if socket_options is None: + socket_options = [] + + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + with map_exceptions(exc_map): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + for option in socket_options: + sock.setsockopt(*option) + sock.settimeout(timeout) + sock.connect(path) + return SyncStream(sock) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/trio.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/trio.py new file mode 100644 index 0000000000000000000000000000000000000000..6f53f5f2a025e01e9949e2530bd9ca6928859251 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/httpcore/_backends/trio.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import ssl +import typing + +import trio + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ExceptionMapping, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class TrioStream(AsyncNetworkStream): + def __init__(self, stream: trio.abc.Stream) -> None: + self._stream = stream + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ReadTimeout, + trio.BrokenResourceError: ReadError, + trio.ClosedResourceError: ReadError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + data: bytes = await self._stream.receive_some(max_bytes=max_bytes) + return data + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: WriteTimeout, + trio.BrokenResourceError: WriteError, + trio.ClosedResourceError: WriteError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + await self._stream.send_all(data=buffer) + + async def aclose(self) -> None: + await self._stream.aclose() + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + } + ssl_stream = trio.SSLStream( + self._stream, + ssl_context=ssl_context, + server_hostname=server_hostname, + https_compatible=True, + server_side=False, + ) + with map_exceptions(exc_map): + try: + with trio.fail_after(timeout_or_inf): + await ssl_stream.do_handshake() + except Exception as exc: # pragma: nocover + await self.aclose() + raise exc + return TrioStream(ssl_stream) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object" and isinstance(self._stream, trio.SSLStream): + # Type checkers cannot see `_ssl_object` attribute because trio._ssl.SSLStream uses __getattr__/__setattr__. + # Tracked at https://github.com/python-trio/trio/issues/542 + return self._stream._ssl_object # type: ignore[attr-defined] + if info == "client_addr": + return self._get_socket_stream().socket.getsockname() + if info == "server_addr": + return self._get_socket_stream().socket.getpeername() + if info == "socket": + stream = self._stream + while isinstance(stream, trio.SSLStream): + stream = stream.transport_stream + assert isinstance(stream, trio.SocketStream) + return stream.socket + if info == "is_readable": + socket = self.get_extra_info("socket") + return socket.is_readable() + return None + + def _get_socket_stream(self) -> trio.SocketStream: + stream = self._stream + while isinstance(stream, trio.SSLStream): + stream = stream.transport_stream + assert isinstance(stream, trio.SocketStream) + return stream + + +class TrioBackend(AsyncNetworkBackend): + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + # By default for TCP sockets, trio enables TCP_NODELAY. + # https://trio.readthedocs.io/en/stable/reference-io.html#trio.SocketStream + if socket_options is None: + socket_options = [] # pragma: no cover + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + OSError: ConnectError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + stream: trio.abc.Stream = await trio.open_tcp_stream( + host=host, port=port, local_address=local_address + ) + for option in socket_options: + stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return TrioStream(stream) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + OSError: ConnectError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + stream: trio.abc.Stream = await trio.open_unix_socket(path) + for option in socket_options: + stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return TrioStream(stream) + + async def sleep(self, seconds: float) -> None: + await trio.sleep(seconds) # pragma: nocover diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbb640e004cef0e950a656a53d92d89d82c7472 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/__init__.py @@ -0,0 +1,18 @@ +__all__ = ( + "decode", + "DECODE_DEFAULT_CHARS", + "DECODE_COMPONENT_CHARS", + "encode", + "ENCODE_DEFAULT_CHARS", + "ENCODE_COMPONENT_CHARS", + "format", + "parse", + "URL", +) +__version__ = "0.1.2" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from mdurl._decode import DECODE_COMPONENT_CHARS, DECODE_DEFAULT_CHARS, decode +from mdurl._encode import ENCODE_COMPONENT_CHARS, ENCODE_DEFAULT_CHARS, encode +from mdurl._format import format +from mdurl._parse import url_parse as parse +from mdurl._url import URL diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_decode.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_decode.py new file mode 100644 index 0000000000000000000000000000000000000000..9b50a2dde976a6d43491ec6f20d12e60f6f6597f --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_decode.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from collections.abc import Sequence +import functools +import re + +DECODE_DEFAULT_CHARS = ";/?:@&=+$,#" +DECODE_COMPONENT_CHARS = "" + +decode_cache: dict[str, list[str]] = {} + + +def get_decode_cache(exclude: str) -> Sequence[str]: + if exclude in decode_cache: + return decode_cache[exclude] + + cache: list[str] = [] + decode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + cache.append(ch) + + for i in range(len(exclude)): + ch_code = ord(exclude[i]) + cache[ch_code] = "%" + ("0" + hex(ch_code)[2:].upper())[-2:] + + return cache + + +# Decode percent-encoded string. +# +def decode(string: str, exclude: str = DECODE_DEFAULT_CHARS) -> str: + cache = get_decode_cache(exclude) + repl_func = functools.partial(repl_func_with_cache, cache=cache) + return re.sub(r"(%[a-f0-9]{2})+", repl_func, string, flags=re.IGNORECASE) + + +def repl_func_with_cache(match: re.Match, cache: Sequence[str]) -> str: + seq = match.group() + result = "" + + i = 0 + l = len(seq) # noqa: E741 + while i < l: + b1 = int(seq[i + 1 : i + 3], 16) + + if b1 < 0x80: + result += cache[b1] + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xE0) == 0xC0 and (i + 3 < l): + # 110xxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + + if (b2 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 2 + + i += 3 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF0) == 0xE0 and (i + 6 < l): + # 1110xxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 3 + + i += 6 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF8) == 0xF0 and (i + 9 < l): + # 111110xx 10xxxxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + b4 = int(seq[i + 10 : i + 12], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80 and (b4 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3, b4)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 4 + + i += 9 + i += 3 # emulate JS for loop statement3 + continue + + result += "\ufffd" + i += 3 # emulate JS for loop statement3 + + return result diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_encode.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_encode.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2e5b917afe9e9ecaa6f11af7a9ac82704d3914 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_encode.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Sequence +from string import ascii_letters, digits, hexdigits +from urllib.parse import quote as encode_uri_component + +ASCII_LETTERS_AND_DIGITS = ascii_letters + digits + +ENCODE_DEFAULT_CHARS = ";/?:@&=+$,-_.!~*'()#" +ENCODE_COMPONENT_CHARS = "-_.!~*'()" + +encode_cache: dict[str, list[str]] = {} + + +# Create a lookup array where anything but characters in `chars` string +# and alphanumeric chars is percent-encoded. +def get_encode_cache(exclude: str) -> Sequence[str]: + if exclude in encode_cache: + return encode_cache[exclude] + + cache: list[str] = [] + encode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + + if ch in ASCII_LETTERS_AND_DIGITS: + # always allow unencoded alphanumeric characters + cache.append(ch) + else: + cache.append("%" + ("0" + hex(i)[2:].upper())[-2:]) + + for i in range(len(exclude)): + cache[ord(exclude[i])] = exclude[i] + + return cache + + +# Encode unsafe characters with percent-encoding, skipping already +# encoded sequences. +# +# - string - string to encode +# - exclude - list of characters to ignore (in addition to a-zA-Z0-9) +# - keepEscaped - don't encode '%' in a correct escape sequence (default: true) +def encode( + string: str, exclude: str = ENCODE_DEFAULT_CHARS, *, keep_escaped: bool = True +) -> str: + result = "" + + cache = get_encode_cache(exclude) + + l = len(string) # noqa: E741 + i = 0 + while i < l: + code = ord(string[i]) + + # % + if keep_escaped and code == 0x25 and i + 2 < l: + if all(c in hexdigits for c in string[i + 1 : i + 3]): + result += string[i : i + 3] + i += 2 + i += 1 # JS for loop statement3 + continue + + if code < 128: + result += cache[code] + i += 1 # JS for loop statement3 + continue + + if code >= 0xD800 and code <= 0xDFFF: + if code >= 0xD800 and code <= 0xDBFF and i + 1 < l: + next_code = ord(string[i + 1]) + if next_code >= 0xDC00 and next_code <= 0xDFFF: + result += encode_uri_component(string[i] + string[i + 1]) + i += 1 + i += 1 # JS for loop statement3 + continue + result += "%EF%BF%BD" + i += 1 # JS for loop statement3 + continue + + result += encode_uri_component(string[i]) + i += 1 # JS for loop statement3 + + return result diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_format.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_format.py new file mode 100644 index 0000000000000000000000000000000000000000..12524ca626065183ec9974f3d7d08dadd4a7d3e8 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_format.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from mdurl._url import URL + + +def format(url: URL) -> str: # noqa: A001 + result = "" + + result += url.protocol or "" + result += "//" if url.slashes else "" + result += url.auth + "@" if url.auth else "" + + if url.hostname and ":" in url.hostname: + # ipv6 address + result += "[" + url.hostname + "]" + else: + result += url.hostname or "" + + result += ":" + url.port if url.port else "" + result += url.pathname or "" + result += url.search or "" + result += url.hash or "" + + return result diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_parse.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_parse.py new file mode 100644 index 0000000000000000000000000000000000000000..ffeeac768dca3bff60c55c9b1f0bc0fbb4cec7b1 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_parse.py @@ -0,0 +1,304 @@ +# Copyright Joyent, Inc. and other Node contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# Changes from joyent/node: +# +# 1. No leading slash in paths, +# e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/` +# +# 2. Backslashes are not replaced with slashes, +# so `http:\\example.org\` is treated like a relative path +# +# 3. Trailing colon is treated like a part of the path, +# i.e. in `http://example.org:foo` pathname is `:foo` +# +# 4. Nothing is URL-encoded in the resulting object, +# (in joyent/node some chars in auth and paths are encoded) +# +# 5. `url.parse()` does not have `parseQueryString` argument +# +# 6. Removed extraneous result properties: `host`, `path`, `query`, etc., +# which can be constructed using other parts of the url. + +from __future__ import annotations + +from collections import defaultdict +import re + +from mdurl._url import URL + +# Reference: RFC 3986, RFC 1808, RFC 2396 + +# define these here so at least they only have to be +# compiled once on the first module load. +PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE) +PORT_PATTERN = re.compile(r":[0-9]*$") + +# Special case for a simple path URL +SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$") + +# RFC 2396: characters reserved for delimiting URLs. +# We actually just auto-escape these. +DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t") + +# RFC 2396: characters not allowed for various reasons. +UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS + +# Allowed by RFCs, but cause of XSS attacks. Always escape these. +AUTO_ESCAPE = ("'",) + UNWISE +# Characters that are never ever allowed in a hostname. +# Note that any invalid chars are also handled, but these +# are the ones that are *expected* to be seen, so we fast-path +# them. +NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE +HOST_ENDING_CHARS = ("/", "?", "#") +HOSTNAME_MAX_LEN = 255 +HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$") +HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$") +# protocols that can allow "unsafe" and "unwise" chars. + +# protocols that never have a hostname. +HOSTLESS_PROTOCOL = defaultdict( + bool, + { + "javascript": True, + "javascript:": True, + }, +) +# protocols that always contain a // bit. +SLASHED_PROTOCOL = defaultdict( + bool, + { + "http": True, + "https": True, + "ftp": True, + "gopher": True, + "file": True, + "http:": True, + "https:": True, + "ftp:": True, + "gopher:": True, + "file:": True, + }, +) + + +class MutableURL: + def __init__(self) -> None: + self.protocol: str | None = None + self.slashes: bool = False + self.auth: str | None = None + self.port: str | None = None + self.hostname: str | None = None + self.hash: str | None = None + self.search: str | None = None + self.pathname: str | None = None + + def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL": + lower_proto = "" + slashes = False + rest = url + + # trim before proceeding. + # This is to support parse stuff like " http://foo.com \n" + rest = rest.strip() + + if not slashes_denote_host and len(url.split("#")) == 1: + # Try fast path regexp + simple_path = SIMPLE_PATH_PATTERN.match(rest) + if simple_path: + self.pathname = simple_path.group(1) + if simple_path.group(2): + self.search = simple_path.group(2) + return self + + proto = "" + proto_match = PROTOCOL_PATTERN.match(rest) + if proto_match: + proto = proto_match.group() + lower_proto = proto.lower() + self.protocol = proto + rest = rest[len(proto) :] + + # figure out if it's got a host + # user@server is *always* interpreted as a hostname, and url + # resolution will treat //foo/bar as host=foo,path=bar because that's + # how the browser resolves relative URLs. + if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest): + slashes = rest.startswith("//") + if slashes and not (proto and HOSTLESS_PROTOCOL[proto]): + rest = rest[2:] + self.slashes = True + + if not HOSTLESS_PROTOCOL[proto] and ( + slashes or (proto and not SLASHED_PROTOCOL[proto]) + ): + + # there's a hostname. + # the first instance of /, ?, ;, or # ends the host. + # + # If there is an @ in the hostname, then non-host chars *are* allowed + # to the left of the last @ sign, unless some host-ending character + # comes *before* the @-sign. + # URLs are obnoxious. + # + # ex: + # http://a@b@c/ => user:a@b host:c + # http://a@b?@c => user:a host:c path:/?@c + + # v0.12 TODO(isaacs): This is not quite how Chrome does things. + # Review our test case against browsers more comprehensively. + + # find the first instance of any hostEndingChars + host_end = -1 + for i in range(len(HOST_ENDING_CHARS)): + hec = rest.find(HOST_ENDING_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + + # at this point, either we have an explicit point where the + # auth portion cannot go past, or the last @ char is the decider. + if host_end == -1: + # atSign can be anywhere. + at_sign = rest.rfind("@") + else: + # atSign must be in auth portion. + # http://a@b/c@d => host:b auth:a path:/c@d + at_sign = rest.rfind("@", 0, host_end + 1) + + # Now we have a portion which is definitely the auth. + # Pull that off. + if at_sign != -1: + auth = rest[:at_sign] + rest = rest[at_sign + 1 :] + self.auth = auth + + # the host is the remaining to the left of the first non-host char + host_end = -1 + for i in range(len(NON_HOST_CHARS)): + hec = rest.find(NON_HOST_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + # if we still have not hit it, then the entire thing is a host. + if host_end == -1: + host_end = len(rest) + + if host_end > 0 and rest[host_end - 1] == ":": + host_end -= 1 + host = rest[:host_end] + rest = rest[host_end:] + + # pull out port. + self.parse_host(host) + + # we've indicated that there is a hostname, + # so even if it's empty, it has to be present. + self.hostname = self.hostname or "" + + # if hostname begins with [ and ends with ] + # assume that it's an IPv6 address. + ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith( + "]" + ) + + # validate a little. + if not ipv6_hostname: + hostparts = self.hostname.split(".") + l = len(hostparts) # noqa: E741 + i = 0 + while i < l: + part = hostparts[i] + if not part: + i += 1 # emulate statement3 in JS for loop + continue + if not HOSTNAME_PART_PATTERN.search(part): + newpart = "" + k = len(part) + j = 0 + while j < k: + if ord(part[j]) > 127: + # we replace non-ASCII char with a temporary placeholder + # we need this to make sure size of hostname is not + # broken by replacing non-ASCII by nothing + newpart += "x" + else: + newpart += part[j] + j += 1 # emulate statement3 in JS for loop + + # we test again with ASCII char only + if not HOSTNAME_PART_PATTERN.search(newpart): + valid_parts = hostparts[:i] + not_host = hostparts[i + 1 :] + bit = HOSTNAME_PART_START.search(part) + if bit: + valid_parts.append(bit.group(1)) + not_host.insert(0, bit.group(2)) + if not_host: + rest = ".".join(not_host) + rest + self.hostname = ".".join(valid_parts) + break + i += 1 # emulate statement3 in JS for loop + + if len(self.hostname) > HOSTNAME_MAX_LEN: + self.hostname = "" + + # strip [ and ] from the hostname + # the host field still retains them, though + if ipv6_hostname: + self.hostname = self.hostname[1:-1] + + # chop off from the tail first. + hash = rest.find("#") # noqa: A001 + if hash != -1: + # got a fragment string. + self.hash = rest[hash:] + rest = rest[:hash] + qm = rest.find("?") + if qm != -1: + self.search = rest[qm:] + rest = rest[:qm] + if rest: + self.pathname = rest + if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname: + self.pathname = "" + + return self + + def parse_host(self, host: str) -> None: + port_match = PORT_PATTERN.search(host) + if port_match: + port = port_match.group() + if port != ":": + self.port = port[1:] + host = host[: -len(port)] + if host: + self.hostname = host + + +def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL: + if isinstance(url, URL): + return url + u = MutableURL() + u.parse(url, slashes_denote_host) + return URL( + u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname + ) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_url.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_url.py new file mode 100644 index 0000000000000000000000000000000000000000..f866e7a179c8854e37c9bba6294f48681e5d99d7 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/_url.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import NamedTuple + + +class URL(NamedTuple): + protocol: str | None + slashes: bool + auth: str | None + port: str | None + hostname: str | None + hash: str | None # noqa: A003 + search: str | None + pathname: str | None diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/py.typed b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..7632ecf77545c5e5501cb3fc5719df0761104ca2 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/mdurl/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/INSTALLER b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/LICENSE.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d226599fd2ec2f3353ffb0c434cd8be1b570ac95 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/LICENSE.txt @@ -0,0 +1,971 @@ +Copyright (c) 2005-2023, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---- + +The NumPy repository and source distributions bundle several libraries that are +compatibly licensed. We list these here. + +Name: lapack-lite +Files: numpy/linalg/lapack_lite/* +License: BSD-3-Clause + For details, see numpy/linalg/lapack_lite/LICENSE.txt + +Name: tempita +Files: tools/npy_tempita/* +License: MIT + For details, see tools/npy_tempita/license.txt + +Name: dragon4 +Files: numpy/core/src/multiarray/dragon4.c +License: MIT + For license text, see numpy/core/src/multiarray/dragon4.c + +Name: libdivide +Files: numpy/core/include/numpy/libdivide/* +License: Zlib + For license text, see numpy/core/include/numpy/libdivide/LICENSE.txt + + +Note that the following files are vendored in the repository and sdist but not +installed in built numpy packages: + +Name: Meson +Files: vendored-meson/meson/* +License: Apache 2.0 + For license text, see vendored-meson/meson/COPYING + +Name: spin +Files: .spin/cmds.py +License: BSD-3 + For license text, see .spin/LICENSE + +---- + +This binary distribution of NumPy also bundles the following software: + + +Name: OpenBLAS +Files: numpy.libs/libopenblas*.so +Description: bundled as a dynamically linked library +Availability: https://github.com/OpenMathLib/OpenBLAS/ +License: BSD-3-Clause + Copyright (c) 2011-2014, The OpenBLAS Project + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: LAPACK +Files: numpy.libs/libopenblas*.so +Description: bundled in OpenBLAS +Availability: https://github.com/OpenMathLib/OpenBLAS/ +License: BSD-3-Clause-Attribution + Copyright (c) 1992-2013 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. + Copyright (c) 2000-2013 The University of California Berkeley. All + rights reserved. + Copyright (c) 2006-2013 The University of Colorado Denver. All rights + reserved. + + $COPYRIGHT$ + + Additional copyrights may follow + + $HEADER$ + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + + - Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + The copyright holders provide no reassurances that the source code + provided does not infringe any patent, copyright, or any other + intellectual property rights of third parties. The copyright holders + disclaim any liability to any recipient for claims brought against + recipient by any third party for infringement of that parties + intellectual property rights. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: GCC runtime library +Files: numpy.libs/libgfortran*.so +Description: dynamically linked to files compiled with gcc +Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libgfortran +License: GPL-3.0-with-GCC-exception + Copyright (C) 2002-2017 Free Software Foundation, Inc. + + Libgfortran is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgfortran is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . + +---- + +Full text of license texts referred to above follows (that they are +listed below does not necessarily imply the conditions apply to the +present binary release): + +---- + +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. + +---- + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + +Name: libquadmath +Files: numpy.libs/libquadmath*.so +Description: dynamically linked to files compiled with gcc +Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libquadmath +License: LGPL-2.1-or-later + + GCC Quad-Precision Math Library + Copyright (C) 2010-2019 Free Software Foundation, Inc. + Written by Francois-Xavier Coudert + + This file is part of the libquadmath library. + Libquadmath is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + Libquadmath is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/METADATA b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8246dc4ed3bfc5e7ea1f402d2aff1b74ca62e16f --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/METADATA @@ -0,0 +1,1092 @@ +Metadata-Version: 2.1 +Name: numpy +Version: 1.26.4 +Summary: Fundamental package for array computing in Python +Home-page: https://numpy.org +Author: Travis E. Oliphant et al. +Maintainer-Email: NumPy Developers +License: Copyright (c) 2005-2023, NumPy Developers. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ---- + + The NumPy repository and source distributions bundle several libraries that are + compatibly licensed. We list these here. + + Name: lapack-lite + Files: numpy/linalg/lapack_lite/* + License: BSD-3-Clause + For details, see numpy/linalg/lapack_lite/LICENSE.txt + + Name: tempita + Files: tools/npy_tempita/* + License: MIT + For details, see tools/npy_tempita/license.txt + + Name: dragon4 + Files: numpy/core/src/multiarray/dragon4.c + License: MIT + For license text, see numpy/core/src/multiarray/dragon4.c + + Name: libdivide + Files: numpy/core/include/numpy/libdivide/* + License: Zlib + For license text, see numpy/core/include/numpy/libdivide/LICENSE.txt + + + Note that the following files are vendored in the repository and sdist but not + installed in built numpy packages: + + Name: Meson + Files: vendored-meson/meson/* + License: Apache 2.0 + For license text, see vendored-meson/meson/COPYING + + Name: spin + Files: .spin/cmds.py + License: BSD-3 + For license text, see .spin/LICENSE + + ---- + + This binary distribution of NumPy also bundles the following software: + + + Name: OpenBLAS + Files: numpy.libs/libopenblas*.so + Description: bundled as a dynamically linked library + Availability: https://github.com/OpenMathLib/OpenBLAS/ + License: BSD-3-Clause + Copyright (c) 2011-2014, The OpenBLAS Project + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + Name: LAPACK + Files: numpy.libs/libopenblas*.so + Description: bundled in OpenBLAS + Availability: https://github.com/OpenMathLib/OpenBLAS/ + License: BSD-3-Clause-Attribution + Copyright (c) 1992-2013 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. + Copyright (c) 2000-2013 The University of California Berkeley. All + rights reserved. + Copyright (c) 2006-2013 The University of Colorado Denver. All rights + reserved. + + $COPYRIGHT$ + + Additional copyrights may follow + + $HEADER$ + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + + - Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + The copyright holders provide no reassurances that the source code + provided does not infringe any patent, copyright, or any other + intellectual property rights of third parties. The copyright holders + disclaim any liability to any recipient for claims brought against + recipient by any third party for infringement of that parties + intellectual property rights. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + Name: GCC runtime library + Files: numpy.libs/libgfortran*.so + Description: dynamically linked to files compiled with gcc + Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libgfortran + License: GPL-3.0-with-GCC-exception + Copyright (C) 2002-2017 Free Software Foundation, Inc. + + Libgfortran is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgfortran is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . + + ---- + + Full text of license texts referred to above follows (that they are + listed below does not necessarily imply the conditions apply to the + present binary release): + + ---- + + GCC RUNTIME LIBRARY EXCEPTION + + Version 3.1, 31 March 2009 + + Copyright (C) 2009 Free Software Foundation, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + This GCC Runtime Library Exception ("Exception") is an additional + permission under section 7 of the GNU General Public License, version + 3 ("GPLv3"). It applies to a given file (the "Runtime Library") that + bears a notice placed by the copyright holder of the file stating that + the file is governed by GPLv3 along with this Exception. + + When you use GCC to compile a program, GCC may combine portions of + certain GCC header files and runtime libraries with the compiled + program. The purpose of this Exception is to allow compilation of + non-GPL (including proprietary) programs to use, in this way, the + header files and runtime libraries covered by this Exception. + + 0. Definitions. + + A file is an "Independent Module" if it either requires the Runtime + Library for execution after a Compilation Process, or makes use of an + interface provided by the Runtime Library, but is not otherwise based + on the Runtime Library. + + "GCC" means a version of the GNU Compiler Collection, with or without + modifications, governed by version 3 (or a specified later version) of + the GNU General Public License (GPL) with the option of using any + subsequent versions published by the FSF. + + "GPL-compatible Software" is software whose conditions of propagation, + modification and use would permit combination with GCC in accord with + the license of GCC. + + "Target Code" refers to output from any compiler for a real or virtual + target processor architecture, in executable form or suitable for + input to an assembler, loader, linker and/or execution + phase. Notwithstanding that, Target Code does not include data in any + format that is used as a compiler intermediate representation, or used + for producing a compiler intermediate representation. + + The "Compilation Process" transforms code entirely represented in + non-intermediate languages designed for human-written code, and/or in + Java Virtual Machine byte code, into Target Code. Thus, for example, + use of source code generators and preprocessors need not be considered + part of the Compilation Process, since the Compilation Process can be + understood as starting with the output of the generators or + preprocessors. + + A Compilation Process is "Eligible" if it is done using GCC, alone or + with other GPL-compatible software, or if it is done without using any + work based on GCC. For example, using non-GPL-compatible Software to + optimize any GCC intermediate representations would not qualify as an + Eligible Compilation Process. + + 1. Grant of Additional Permission. + + You have permission to propagate a work of Target Code formed by + combining the Runtime Library with Independent Modules, even if such + propagation would otherwise violate the terms of GPLv3, provided that + all Target Code was generated by Eligible Compilation Processes. You + may then convey such a combination under terms of your choice, + consistent with the licensing of the Independent Modules. + + 2. No Weakening of GCC Copyleft. + + The availability of this Exception does not imply any general + presumption that third-party software is unaffected by the copyleft + requirements of the license of GCC. + + ---- + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for + software and other kinds of works. + + The licenses for most software and other practical works are designed + to take away your freedom to share and change the works. By contrast, + the GNU General Public License is intended to guarantee your freedom to + share and change all versions of a program--to make sure it remains free + software for all its users. We, the Free Software Foundation, use the + GNU General Public License for most of our software; it applies also to + any other work released this way by its authors. You can apply it to + your programs, too. + + When we speak of free software, we are referring to freedom, not + price. Our General Public Licenses are designed to make sure that you + have the freedom to distribute copies of free software (and charge for + them if you wish), that you receive source code or can get it if you + want it, that you can change the software or use pieces of it in new + free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you + these rights or asking you to surrender the rights. Therefore, you have + certain responsibilities if you distribute copies of the software, or if + you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether + gratis or for a fee, you must pass on to the recipients the same + freedoms that you received. You must make sure that they, too, receive + or can get the source code. And you must show them these terms so they + know their rights. + + Developers that use the GNU GPL protect your rights with two steps: + (1) assert copyright on the software, and (2) offer you this License + giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains + that there is no warranty for this free software. For both users' and + authors' sake, the GPL requires that modified versions be marked as + changed, so that their problems will not be attributed erroneously to + authors of previous versions. + + Some devices are designed to deny users access to install or run + modified versions of the software inside them, although the manufacturer + can do so. This is fundamentally incompatible with the aim of + protecting users' freedom to change the software. The systematic + pattern of such abuse occurs in the area of products for individuals to + use, which is precisely where it is most unacceptable. Therefore, we + have designed this version of the GPL to prohibit the practice for those + products. If such problems arise substantially in other domains, we + stand ready to extend this provision to those domains in future versions + of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. + States should not allow patents to restrict development and use of + software on general-purpose computers, but in those that do, we wish to + avoid the special danger that patents applied to a free program could + make it effectively proprietary. To prevent this, the GPL assures that + patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and + modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this + License. Each licensee is addressed as "you". "Licensees" and + "recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work + in a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a "modified version" of the + earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based + on the Program. + + To "propagate" a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through + a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" + to the extent that it includes a convenient and prominently visible + feature that (1) displays an appropriate copyright notice, and (2) + tells the user that there is no warranty for the work (except to the + extent that warranties are provided), that licensees may convey the + work under this License, and how to view a copy of this License. If + the interface presents a list of user commands or options, such as a + menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work + for making modifications to it. "Object code" means any non-source + form of a work. + + A "Standard Interface" means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that + is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other + than the work as a whole, that (a) is included in the normal form of + packaging a Major Component, but which is not part of that Major + Component, and (b) serves only to enable use of the work with that + Major Component, or to implement a Standard Interface for which an + implementation is available to the public in source code form. A + "Major Component", in this context, means a major essential component + (kernel, window system, and so on) of the specific operating system + (if any) on which the executable work runs, or a compiler used to + produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all + the source code needed to generate, install, and (for an executable + work) run the object code and to modify the work, including scripts to + control those activities. However, it does not include the work's + System Libraries, or general-purpose tools or generally available free + programs which are used unmodified in performing those activities but + which are not part of the work. For example, Corresponding Source + includes interface definition files associated with source files for + the work, and the source code for shared libraries and dynamically + linked subprograms that the work is specifically designed to require, + such as by intimate data communication or control flow between those + subprograms and other parts of the work. + + The Corresponding Source need not include anything that users + can regenerate automatically from other parts of the Corresponding + Source. + + The Corresponding Source for a work in source code form is that + same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program. The output from running a + covered work is covered by this License only if the output, given its + content, constitutes a covered work. This License acknowledges your + rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not + convey, without conditions so long as your license otherwise remains + in force. You may convey covered works to others for the sole purpose + of having them make modifications exclusively for you, or provide you + with facilities for running those works, provided that you comply with + the terms of this License in conveying all material for which you do + not control copyright. Those thus making or running the covered works + for you must do so exclusively on your behalf, under your direction + and control, on terms that prohibit them from making any copies of + your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under + the conditions stated below. Sublicensing is not allowed; section 10 + makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article + 11 of the WIPO copyright treaty adopted on 20 December 1996, or + similar laws prohibiting or restricting circumvention of such + measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention + is effected by exercising rights under this License with respect to + the covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's + users, your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; + keep intact all notices stating that this License and any + non-permissive terms added in accord with section 7 apply to the code; + keep intact all notices of the absence of any warranty; and give all + recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, + and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the + terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, + and which are not combined with it such as to form a larger program, + in or on a volume of a storage or distribution medium, is called an + "aggregate" if the compilation and its resulting copyright are not + used to limit the access or legal rights of the compilation's users + beyond what the individual works permit. Inclusion of a covered work + in an aggregate does not cause this License to apply to the other + parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms + of sections 4 and 5, provided that you also convey the + machine-readable Corresponding Source under the terms of this License, + in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be + included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, "normally used" refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as + part of a transaction in which the right of possession and use of the + User Product is transferred to the recipient in perpetuity or for a + fixed term (regardless of how the transaction is characterized), the + Corresponding Source conveyed under this section must be accompanied + by the Installation Information. But this requirement does not apply + if neither you nor any third party retains the ability to install + modified object code on the User Product (for example, the work has + been installed in ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access to a + network may be denied when the modification itself materially and + adversely affects the operation of the network or violates the rules and + protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, + in accord with this section must be in a format that is publicly + documented (and with an implementation available to the public in + source code form), and must require no special password or key for + unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall + be treated as though they were included in this License, to the extent + that they are valid under applicable law. If additional permissions + apply only to part of the Program, that part may be used separately + under those permissions, but the entire Program remains governed by + this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option + remove any additional permissions from that copy, or from any part of + it. (Additional permissions may be written to require their own + removal in certain cases when you modify the work.) You may place + additional permissions on material, added by you to a covered work, + for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you + add to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further + restrictions" within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further + restriction, you may remove that term. If a license document contains + a further restriction but permits relicensing or conveying under this + License, you may add to a covered work material governed by the terms + of that license document, provided that the further restriction does + not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you + must place, in the relevant source files, a statement of the + additional terms that apply to those files, or a notice indicating + where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the + form of a separately written license, or stated as exceptions; + the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or + modify it is void, and will automatically terminate your rights under + this License (including any patent licenses granted under the third + paragraph of section 11). + + However, if you cease all violation of this License, then your + license from a particular copyright holder is reinstated (a) + provisionally, unless and until the copyright holder explicitly and + finally terminates your license, and (b) permanently, if the copyright + holder fails to notify you of the violation by some reasonable means + prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is + reinstated permanently if the copyright holder notifies you of the + violation by some reasonable means, this is the first time you have + received notice of violation of this License (for any work) from that + copyright holder, and you cure the violation prior to 30 days after + your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or + run a copy of the Program. Ancillary propagation of a covered work + occurring solely as a consequence of using peer-to-peer transmission + to receive a copy likewise does not require acceptance. However, + nothing other than this License grants you permission to propagate or + modify any covered work. These actions infringe copyright if you do + not accept this License. Therefore, by modifying or propagating a + covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically + receives a license from the original licensors, to run, modify and + propagate that work, subject to this License. You are not responsible + for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered + work results from an entity transaction, each party to that + transaction who receives a copy of the work also receives whatever + licenses to the work the party's predecessor in interest had or could + give under the previous paragraph, plus a right to possession of the + Corresponding Source of the work from the predecessor in interest, if + the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the + rights granted or affirmed under this License. For example, you may + not impose a license fee, royalty, or other charge for exercise of + rights granted under this License, and you may not initiate litigation + (including a cross-claim or counterclaim in a lawsuit) alleging that + any patent claim is infringed by making, using, selling, offering for + sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The + work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims + owned or controlled by the contributor, whether already acquired or + hereafter acquired, that would be infringed by some manner, permitted + by this License, of making, using, or selling its contributor version, + but do not include claims that would be infringed only as a + consequence of further modification of the contributor version. For + purposes of this definition, "control" includes the right to grant + patent sublicenses in a manner consistent with the requirements of + this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to + make, use, sell, offer for sale, import and otherwise run, modify and + propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To "grant" such a patent license to a + party means to make such an agreement or commitment not to enforce a + patent against the party. + + If you convey a covered work, knowingly relying on a patent license, + and the Corresponding Source of the work is not available for anyone + to copy, free of charge and under the terms of this License, through a + publicly available network server or other readily accessible means, + then you must either (1) cause the Corresponding Source to be so + available, or (2) arrange to deprive yourself of the benefit of the + patent license for this particular work, or (3) arrange, in a manner + consistent with the requirements of this License, to extend the patent + license to downstream recipients. "Knowingly relying" means you have + actual knowledge that, but for the patent license, your conveying the + covered work in a country, or your recipient's use of the covered work + in a country, would infringe one or more identifiable patents in that + country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties + receiving the covered work authorizing them to use, propagate, modify + or convey a specific copy of the covered work, then the patent license + you grant is automatically extended to all recipients of the covered + work and works based on it. + + A patent license is "discriminatory" if it does not include within + the scope of its coverage, prohibits the exercise of, or is + conditioned on the non-exercise of one or more of the rights that are + specifically granted under this License. You may not convey a covered + work if you are a party to an arrangement with a third party that is + in the business of distributing software, under which you make payment + to the third party based on the extent of your activity of conveying + the work, and under which the third party grants, to any of the + parties who would receive the covered work from you, a discriminatory + patent license (a) in connection with copies of the covered work + conveyed by you (or copies made from those copies), or (b) primarily + for and in connection with specific products or compilations that + contain the covered work, unless you entered into that arrangement, + or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting + any implied license or other defenses to infringement that may + otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot convey a + covered work so as to satisfy simultaneously your obligations under this + License and any other pertinent obligations, then as a consequence you may + not convey it at all. For example, if you agree to terms that obligate you + to collect a royalty for further conveying from those to whom you convey + the Program, the only way you could satisfy both those terms and this + License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have + permission to link or combine any covered work with a work licensed + under version 3 of the GNU Affero General Public License into a single + combined work, and to convey the resulting work. The terms of this + License will continue to apply to the part which is the covered work, + but the special requirements of the GNU Affero General Public License, + section 13, concerning interaction through a network will apply to the + combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of + the GNU General Public License from time to time. Such new versions will + be similar in spirit to the present version, but may differ in detail to + address new problems or concerns. + + Each version is given a distinguishing version number. If the + Program specifies that a certain numbered version of the GNU General + Public License "or any later version" applies to it, you have the + option of following the terms and conditions either of that numbered + version or of any later version published by the Free Software + Foundation. If the Program does not specify a version number of the + GNU General Public License, you may choose any version ever published + by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future + versions of the GNU General Public License can be used, that proxy's + public statement of acceptance of a version permanently authorizes you + to choose that version for the Program. + + Later license versions may give you additional or different + permissions. However, no additional obligations are imposed on any + author or copyright holder as a result of your choosing to follow a + later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY + GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE + USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF + DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD + PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), + EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF + SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided + above cannot be given local legal effect according to their terms, + reviewing courts shall apply local law that most closely approximates + an absolute waiver of all civil liability in connection with the + Program, unless a warranty or assumption of liability accompanies a + copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest + to attach them to the start of each source file to most effectively + state the exclusion of warranty; and each file should have at least + the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + + Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short + notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the appropriate + parts of the General Public License. Of course, your program's commands + might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, + if any, to sign a "copyright disclaimer" for the program, if necessary. + For more information on this, and how to apply and follow the GNU GPL, see + . + + The GNU General Public License does not permit incorporating your program + into proprietary programs. If your program is a subroutine library, you + may consider it more useful to permit linking proprietary applications with + the library. If this is what you want to do, use the GNU Lesser General + Public License instead of this License. But first, please read + . + + Name: libquadmath + Files: numpy.libs/libquadmath*.so + Description: dynamically linked to files compiled with gcc + Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libquadmath + License: LGPL-2.1-or-later + + GCC Quad-Precision Math Library + Copyright (C) 2010-2019 Free Software Foundation, Inc. + Written by Francois-Xavier Coudert + + This file is part of the libquadmath library. + Libquadmath is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + Libquadmath is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Science/Research +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: C +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: Software Development +Classifier: Topic :: Scientific/Engineering +Classifier: Typing :: Typed +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Operating System :: MacOS +Project-URL: Homepage, https://numpy.org +Project-URL: Documentation, https://numpy.org/doc/ +Project-URL: Source, https://github.com/numpy/numpy +Project-URL: Download, https://pypi.org/project/numpy/#files +Project-URL: Tracker, https://github.com/numpy/numpy/issues +Project-URL: Release notes, https://numpy.org/doc/stable/release +Requires-Python: >=3.9 +Description-Content-Type: text/markdown + +

+ +


+ + +[![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)]( +https://numfocus.org) +[![PyPI Downloads](https://img.shields.io/pypi/dm/numpy.svg?label=PyPI%20downloads)]( +https://pypi.org/project/numpy/) +[![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/numpy.svg?label=Conda%20downloads)]( +https://anaconda.org/conda-forge/numpy) +[![Stack Overflow](https://img.shields.io/badge/stackoverflow-Ask%20questions-blue.svg)]( +https://stackoverflow.com/questions/tagged/numpy) +[![Nature Paper](https://img.shields.io/badge/DOI-10.1038%2Fs41592--019--0686--2-blue)]( +https://doi.org/10.1038/s41586-020-2649-2) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/numpy/numpy/badge)](https://api.securityscorecards.dev/projects/github.com/numpy/numpy) + + +NumPy is the fundamental package for scientific computing with Python. + +- **Website:** https://www.numpy.org +- **Documentation:** https://numpy.org/doc +- **Mailing list:** https://mail.python.org/mailman/listinfo/numpy-discussion +- **Source code:** https://github.com/numpy/numpy +- **Contributing:** https://www.numpy.org/devdocs/dev/index.html +- **Bug reports:** https://github.com/numpy/numpy/issues +- **Report a security vulnerability:** https://tidelift.com/docs/security + +It provides: + +- a powerful N-dimensional array object +- sophisticated (broadcasting) functions +- tools for integrating C/C++ and Fortran code +- useful linear algebra, Fourier transform, and random number capabilities + +Testing: + +NumPy requires `pytest` and `hypothesis`. Tests can then be run after installation with: + + python -c "import numpy, sys; sys.exit(numpy.test() is False)" + +Code of Conduct +---------------------- + +NumPy is a community-driven open source project developed by a diverse group of +[contributors](https://numpy.org/teams/). The NumPy leadership has made a strong +commitment to creating an open, inclusive, and positive community. Please read the +[NumPy Code of Conduct](https://numpy.org/code-of-conduct/) for guidance on how to interact +with others in a way that makes our community thrive. + +Call for Contributions +---------------------- + +The NumPy project welcomes your expertise and enthusiasm! + +Small improvements or fixes are always appreciated. If you are considering larger contributions +to the source code, please contact us through the [mailing +list](https://mail.python.org/mailman/listinfo/numpy-discussion) first. + +Writing code isn’t the only way to contribute to NumPy. You can also: +- review pull requests +- help us stay on top of new and old issues +- develop tutorials, presentations, and other educational materials +- maintain and improve [our website](https://github.com/numpy/numpy.org) +- develop graphic design for our brand assets and promotional materials +- translate website content +- help with outreach and onboard new contributors +- write grant proposals and help with other fundraising efforts + +For more information about the ways you can contribute to NumPy, visit [our website](https://numpy.org/contribute/). +If you’re unsure where to start or how your skills fit in, reach out! You can +ask on the mailing list or here, on GitHub, by opening a new issue or leaving a +comment on a relevant issue that is already open. + +Our preferred channels of communication are all public, but if you’d like to +speak to us in private first, contact our community coordinators at +numpy-team@googlegroups.com or on Slack (write numpy-team@googlegroups.com for +an invitation). + +We also have a biweekly community call, details of which are announced on the +mailing list. You are very welcome to join. + +If you are new to contributing to open source, [this +guide](https://opensource.guide/how-to-contribute/) helps explain why, what, +and how to successfully get involved. diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/RECORD b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..09b9cef54427259f36c7a1d7a6414109397dc083 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/RECORD @@ -0,0 +1,792 @@ +../../../bin/f2py,sha256=ScMFUWEA5j-JCSbPSeQ-eEGgHxanaIyHWKHzNTwpC6A,398 +numpy-1.26.4.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +numpy-1.26.4.dist-info/LICENSE.txt,sha256=EQewyDHpGNTx28KKMxkMdyFe8njUpMQAlXIIh3DUM0o,47721 +numpy-1.26.4.dist-info/METADATA,sha256=sJc0p_7UToS0yBYZNM5TLf8ed57Ggi1BVkTRF_Y4EHA,61041 +numpy-1.26.4.dist-info/RECORD,, +numpy-1.26.4.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy-1.26.4.dist-info/WHEEL,sha256=3qIDcXCk577AXiK3pDifO-gE9U_MYWYGgtD78gLa2_U,137 +numpy-1.26.4.dist-info/entry_points.txt,sha256=zddyYJuUw9Uud7LeLfynXk62_ry0lGihDwCIgugBdZM,144 +numpy.libs/libgfortran-040039e1.so.5.0.0,sha256=FK-zEpsai1C8QKOwggx_EVLqm8EBIaqxUpQ_cFdHKIY,2686065 +numpy.libs/libopenblas64_p-r0-0cf96a72.3.23.dev.so,sha256=klTQhU3XYV4R3ijXca5AiHjKgSOnrCBPIeTMejdswuU,35123345 +numpy.libs/libquadmath-96973f99.so.0.0.0,sha256=k0wi3tDn0WnE1GeIdslgUa3z2UVF2pYvYLQWWbB12js,247609 +numpy/__config__.py,sha256=z0NFqd9D20ShQlKyPTlbfAPWIJFDEJ7aVp3TQ5_vTxU,4902 +numpy/__init__.cython-30.pxd,sha256=yk2a3etxRNlBgj5uLfIho2RYDYDzhRW8oagAG-wzbPI,36690 +numpy/__init__.pxd,sha256=Pa0VYRSeQRSFepQ6ROgZrNtGY5TzBXIddWsMHtK0OkM,35066 +numpy/__init__.py,sha256=Is0VNfoU10729FfMoUn_3ICHX0YL4xO4-JUnP3i8QC4,17005 +numpy/__init__.pyi,sha256=9kK465XL9oS_X3fJLv0Na29NEYnWvtdMhXPtrnF_cG8,154080 +numpy/_core/__init__.py,sha256=C8_7wbHqUkB35JouY_XKsas1KLpRZ7JHWuZ7VGOPVpU,136 +numpy/_core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/_core/_dtype.py,sha256=vE16-yiwUSYsAIbq7FlEY1GbXZAp8wjADDxJg3eBX-U,126 +numpy/_core/_dtype_ctypes.py,sha256=i5EhoWPUhu4kla3Xu4ZvXF1lVLPiI6Zg4h6o8jaiamo,147 +numpy/_core/_internal.py,sha256=g5ugmqDgUhSlie5-onOctcm4p0gcMHSIRLHVYtFTk1M,135 +numpy/_core/_multiarray_umath.py,sha256=VPtoT2uHnyU3rKL0G27CgmNmB1WRHM0mtc7Y9L85C3U,159 +numpy/_core/multiarray.py,sha256=kZxC_7P3Jwz1RApzQU2QGmqSq4MAEvKmaJEYnAsbSOs,138 +numpy/_core/umath.py,sha256=YcV0cdbGcem6D5P3yX7cR9HGYBrT8VMoAgCBzGwPhgg,123 +numpy/_distributor_init.py,sha256=IKy2THwmu5UgBjtVbwbD9H-Ap8uaUJoPJ2btQ4Jatdo,407 +numpy/_globals.py,sha256=neEdcfLZoHLwber_1Xyrn26LcXy0MrSta03Ze7aKa6g,3094 +numpy/_pyinstaller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/_pyinstaller/hook-numpy.py,sha256=PUQ-mNWje6bFALB-mLVFRPkvbM4JpLXunB6sjBbTy5g,1409 +numpy/_pyinstaller/pyinstaller-smoke.py,sha256=6iL-eHMQaG3rxnS5EgcvrCqElm9aKL07Cjr1FZJSXls,1143 +numpy/_pyinstaller/test_pyinstaller.py,sha256=8K-7QxmfoXCG0NwR0bhIgCNrDjGlrTzWnrR1sR8btgU,1135 +numpy/_pytesttester.py,sha256=lQUTvKVz6kT8b4yiMV-uW-vG9KSv9UzqAmxaEMezTd8,6731 +numpy/_pytesttester.pyi,sha256=OtyXSiuSy8o_78w3QNQRjMLpvvNyEdC0aMsx6T-vRxU,489 +numpy/_typing/__init__.py,sha256=6w9E9V9VaT7vTM-veua8XcySv50Je5qSPJzK9HTocIg,7003 +numpy/_typing/_add_docstring.py,sha256=xQhQX372aN_m3XN95CneMxOST2FdPcovR-MXM-9ep58,3922 +numpy/_typing/_array_like.py,sha256=L4gnx2KWG8yYcouz5b9boJIkkFNtOJV6QjcnGCrbnRY,4298 +numpy/_typing/_callable.pyi,sha256=Mf57BwohRn9ye6ixJqjNEnK0gKqnVPE9Gy8vK-6_zxo,11121 +numpy/_typing/_char_codes.py,sha256=LR51O5AUBDbCmJvlMoxyUvsfvb1p7WHrexgtTGtuWTc,5916 +numpy/_typing/_dtype_like.py,sha256=21Uxy0UgIawGM82xjDF_ifMq-nP-Bkhn_LpiK_HvWC4,5661 +numpy/_typing/_extended_precision.py,sha256=dGios-1k-QBGew7YFzONZTzVWxz-aYAaqlccl2_h5Bo,777 +numpy/_typing/_nbit.py,sha256=-EQOShHpB3r30b4RVEcruQRTcTaFAZwtqCJ4BsvpEzA,345 +numpy/_typing/_nested_sequence.py,sha256=5eNaVZAV9tZQLFWHYOuVs336JjoiaWxyZQ7cMKb6m1I,2566 +numpy/_typing/_scalars.py,sha256=eVP8PjlcTIlY7v0fRI3tFXPogWtpLJZ8nFvRRrLjDqs,980 +numpy/_typing/_shape.py,sha256=JPy7jJMkISGFTnkgiEifYM-4xTcjb7JMRkLIIjZLw08,211 +numpy/_typing/_ufunc.pyi,sha256=e74LtOP9e8kkRhvrIJ_RXz9Ua_L43Pd9IixwNwermnM,12638 +numpy/_typing/setup.py,sha256=SE0Q6HPqDjWUfceA4yXgkII8y3z7EiSF0Z-MNwOIyG4,337 +numpy/_utils/__init__.py,sha256=Hhetwsi3eTBe8HdWbG51zXmcrX1DiPLxkYSrslMLYcc,723 +numpy/_utils/_convertions.py,sha256=0xMxdeLOziDmHsRM_8luEh4S-kQdMoMg6GxNDDas69k,329 +numpy/_utils/_inspect.py,sha256=8Ma7QBRwfSWKeK1ShJpFNc7CDhE6fkIE_wr1FxrG1A8,7447 +numpy/_utils/_pep440.py,sha256=Vr7B3QsijR5p6h8YAz2LjNGUyzHUJ5gZ4v26NpZAKDc,14069 +numpy/array_api/__init__.py,sha256=XtttWbDf6Yh0_m4zp-L_us4HKnV3oGwdlB6n-01Q9M8,10375 +numpy/array_api/_array_object.py,sha256=rfCBzE6vUjk4HElQGTVwe6Tw2vxiUx7tmBpQEmm1iBk,43794 +numpy/array_api/_constants.py,sha256=AYayN2jf1Dp5rXZ7WPBdUhtPBo_JMCi-pD9oW5zmFkI,87 +numpy/array_api/_creation_functions.py,sha256=6SqHdzZqHOJFEyWFtqnj6KIKRivrGXxROlgnez_3Mt0,10050 +numpy/array_api/_data_type_functions.py,sha256=P57FOsNdXahNUriVtdldonbvBQrrZkVzxZbcqkR_8AA,6288 +numpy/array_api/_dtypes.py,sha256=kDU1NLvEQN-W2HPmJ2wGPx8jiNkFbrvTCD1T1RT8Pwo,4823 +numpy/array_api/_elementwise_functions.py,sha256=0kGuDX3Ur_Qp6tBMBWTO7LPUxzXNGAlA2SSJhdAp4DU,25992 +numpy/array_api/_indexing_functions.py,sha256=d-gzqzyvR45FQerRYJrbBzCWFnDsZWSI9pggA5QWRO4,715 +numpy/array_api/_manipulation_functions.py,sha256=qCoW5B5FXcFOWKPU9D9MXHdMeXIuzvnHUUvprNlwfjc,3317 +numpy/array_api/_searching_functions.py,sha256=mGZiqheYXGWiDK9rqXFiDKX0_B0mJ1OjdA-9FC2o5lA,1715 +numpy/array_api/_set_functions.py,sha256=ULpfK1zznW9joX1DXSiP0R3ahcDB_po7mZlpsRqi7Fs,2948 +numpy/array_api/_sorting_functions.py,sha256=7pszlxNN7-DNqEZlonGLFQrlXPP7evVA8jN31NShg00,2031 +numpy/array_api/_statistical_functions.py,sha256=HspfYteZWSa3InMs10KZz-sk3ZuW6teX6fNdo829T84,3584 +numpy/array_api/_typing.py,sha256=uKidRp6nYxgHnEPaqXXZsDDZ6tw1LshpbwLvy-09eeM,1347 +numpy/array_api/_utility_functions.py,sha256=HwycylbPAgRVz4nZvjvwqN3mQnJbqKA-NRMaAvIP-CE,824 +numpy/array_api/linalg.py,sha256=QPpG2tG1pZgzjrtTjjOu2GDu3cI6UpSsLrsG_o1jXYk,18411 +numpy/array_api/setup.py,sha256=Wx6qD7GU_APiqKolYPO0OHv4eHGYrjPZmDAgjWhOEhM,341 +numpy/array_api/tests/__init__.py,sha256=t_2GZ3lKcsu4ec4GMKPUDYaeMUJyDquBlQAcPgj7kFE,282 +numpy/array_api/tests/test_array_object.py,sha256=FQoAxP4CLDiv6iih8KKUDSLuYM6dtnDcB1f0pMHw4-M,17035 +numpy/array_api/tests/test_creation_functions.py,sha256=s3A1COWmXIAJdhzd8v7VtL-jbiSspskTqwYy0BTpmpw,5023 +numpy/array_api/tests/test_data_type_functions.py,sha256=qc8ktRlVXWC3PKhxPVWI_UF9f1zZtpmzHjdCtf3e16E,1018 +numpy/array_api/tests/test_elementwise_functions.py,sha256=CTj4LLwtusI51HkpzD0JPohP1ffNxogAVFz8WLuWFzM,3800 +numpy/array_api/tests/test_indexing_functions.py,sha256=AbuBGyEufEAf24b7fy8JQhdJtGPdP9XEIxPTJAfAFFo,627 +numpy/array_api/tests/test_manipulation_functions.py,sha256=wce25dSJjubrGhFxmiatzR_IpmNYp9ICJ9PZBBnZTOQ,1087 +numpy/array_api/tests/test_set_functions.py,sha256=D016G7v3ko49bND5sVERP8IqQXZiwr-2yrKbBPJ-oqg,546 +numpy/array_api/tests/test_sorting_functions.py,sha256=INPiYnuGBcsmWtYqdTTX3ENHmM4iUx4zs9KdwDaSmdA,602 +numpy/array_api/tests/test_validation.py,sha256=QUG9yWC3QhkPxNhbQeakwBbl-0Rr0iTuZ41_0sfVIGU,676 +numpy/compat/__init__.py,sha256=iAHrmsZWzouOMSyD9bdSE0APWMlRpqW92MQgF8y6x3E,448 +numpy/compat/py3k.py,sha256=Je74CVk_7qI_qX7pLbYcuQJsxlMq1poGIfRIrH99kZQ,3833 +numpy/compat/setup.py,sha256=36X1kF0C_NVROXfJ7w3SQeBm5AIDBuJbM5qT7cvSDgU,335 +numpy/compat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/compat/tests/test_compat.py,sha256=YqV67pSN8nXPbXaEdjhmyaoVetNyFupVv57OMEgCwKA,579 +numpy/conftest.py,sha256=HZyWo_wJyrbgnyXxI8t05WOg_IrzNAMnEV7O8koHous,4623 +numpy/core/__init__.py,sha256=CNsO-Ab4ywM2Wz3AbqWOH3ig1q5Bno9PsUMrCv-HNS4,5780 +numpy/core/__init__.pyi,sha256=xtd9OFYza-ZG3jyEJrlzRPT-SkVoB_qYmVCe6FxRks0,126 +numpy/core/_add_newdocs.py,sha256=39JFaeDPN2OQlSwfpY6_Jq9fO5vML8ZMF8J4ZTx_nrs,208972 +numpy/core/_add_newdocs_scalars.py,sha256=PF9v8POcSNH6ELYltkx9e07DWgMmft6NJy9zER3Jk44,12106 +numpy/core/_asarray.py,sha256=P2ddlZAsg1iGleRRfoQv_aKs2N7AGwpo5K4ZQv4Ujlk,3884 +numpy/core/_asarray.pyi,sha256=gNNxUVhToNU_F1QpgeEvUYddpUFN-AKP0QWa4gqcTGw,1086 +numpy/core/_dtype.py,sha256=SihUz41pHRB3Q2LiYYkug6LgMBKh6VV89MOpLxnXQdo,10606 +numpy/core/_dtype_ctypes.py,sha256=Vug4i7xKhznK2tdIjmn4ebclClpaCJwSZUlvEoYl0Eg,3673 +numpy/core/_exceptions.py,sha256=dZWKqfdLRvJvbAEG_fof_8ikEKxjakADMty1kLC_l_M,5379 +numpy/core/_internal.py,sha256=f9kNDuT-FGxF1EtVOVIxXWnH9gM9n-J5V2zwHMv4HEk,28348 +numpy/core/_internal.pyi,sha256=_mCTOX6Su8D4R9fV4HNeohPJx7515B-WOlv4uq6mry8,1032 +numpy/core/_machar.py,sha256=G3a3TXu8VDW_1EMxKKLnGMbvUShEIUEve3ealBlJJ3E,11565 +numpy/core/_methods.py,sha256=m31p0WjcFUGckbJiHnCpSaIQGqv-Lq5niIYkdd33YMo,8613 +numpy/core/_multiarray_tests.cpython-312-x86_64-linux-gnu.so,sha256=Cyy7dBn_wvcSmHqrr1GKOx2d6EBgk_edyx1xKjSrYFc,175912 +numpy/core/_multiarray_umath.cpython-312-x86_64-linux-gnu.so,sha256=amUIEKhzXL25iPdHKZc3QKM3ZF3RWF_vaW5z4tvGW-s,7463681 +numpy/core/_operand_flag_tests.cpython-312-x86_64-linux-gnu.so,sha256=VPbGfwOkzwWoNNVSh3jahuBTI8LrKbN_dCaMcOtDfQE,16856 +numpy/core/_rational_tests.cpython-312-x86_64-linux-gnu.so,sha256=0JmPpR0Ej5eZ4vrHN_6fvrKVCeUVuQam83AxViSkN2k,59776 +numpy/core/_simd.cpython-312-x86_64-linux-gnu.so,sha256=lAK8a8uKjaYoFqMQZBWnVvjeUm-KDsnZzyH_RThl9do,3535232 +numpy/core/_string_helpers.py,sha256=-fQM8z5s8_yX440PmgNEH3SUjEoXMPpPSysZwWZNbuo,2852 +numpy/core/_struct_ufunc_tests.cpython-312-x86_64-linux-gnu.so,sha256=PB6RqEbim2Ezi96GVTzyqi9IuqNcVGCKcgPxwHBVCAM,16960 +numpy/core/_type_aliases.py,sha256=qV6AZlsUWHMWTydmZya73xuBkKXiUKq_WXLj7q2CbZ0,7534 +numpy/core/_type_aliases.pyi,sha256=lguMSqMwvqAFHuRtm8YZSdKbikVz985BdKo_lo7GQCg,404 +numpy/core/_ufunc_config.py,sha256=-Twpe8dnd45ccXH-w-B9nvU8yCOd1E0e3Wpsts3g_bQ,13944 +numpy/core/_ufunc_config.pyi,sha256=-615enOVQMBhVx7Pln7DY_s4H6JjSgSnBy89YkpvuLg,1066 +numpy/core/_umath_tests.cpython-312-x86_64-linux-gnu.so,sha256=kT7z3gJc2t_GgamgqAf3MNRWeVo8KrSWPZVh3mLs_t8,42272 +numpy/core/arrayprint.py,sha256=ySZj4TZFFVCa5yhMmJKFYQYhuQTabZTRBb1YoiCD-ac,63608 +numpy/core/arrayprint.pyi,sha256=21pOWjTSfJOBaKgOOPzRox1ERb3c9ydufqL0b11_P_Q,4428 +numpy/core/cversions.py,sha256=H_iNIpx9-hY1cQNxqjT2d_5SXZhJbMo_caq4_q6LB7I,347 +numpy/core/defchararray.py,sha256=G1LExk-dMeVTYRhtYgcCZEsHk5tkawk7giXcK4Q5KVM,73617 +numpy/core/defchararray.pyi,sha256=ib3aWFcM7F4KooU57mWUNi4GlosNjdfgrLKBVSIKDvU,9216 +numpy/core/einsumfunc.py,sha256=TrL6t79F0H0AQH0y5Cj7Tq0_pzk4fVFi-4q4jJmujYQ,51868 +numpy/core/einsumfunc.pyi,sha256=IJZNdHHG_soig8XvCbXZl43gMr3MMKl9dckTYWecqLs,4860 +numpy/core/fromnumeric.py,sha256=YMtxOBg51VMem39AHXFs-4_vOb1p48ei7njXdYTRJ_Q,128821 +numpy/core/fromnumeric.pyi,sha256=KATMFeFxUJ8YNRaC-jd_dTOt3opz2ng6lHgke5u5COk,23726 +numpy/core/function_base.py,sha256=tHg1qSHTz1eO_wHXNFRt3Q40uqVtPT2eyQdrWbIi4wQ,19836 +numpy/core/function_base.pyi,sha256=3ZYad3cdaGwNEyP8VwK97IYMqk2PDoVjpjQzhIYHjk0,4725 +numpy/core/getlimits.py,sha256=AopcTZDCUXMPcEKIZE1botc3mEhmLb2p1_ejlq1CLqY,25865 +numpy/core/getlimits.pyi,sha256=qeIXUEtognTHr_T-tv-VcZI7n8Z2VzAyIpIgKXzsLkc,82 +numpy/core/include/numpy/__multiarray_api.c,sha256=nPRzTez_Wy3YXy3zZNJNPMspAzxbLOdohqhXwouwMLM,12116 +numpy/core/include/numpy/__multiarray_api.h,sha256=ZM--FKMhIaSQS39cPW0hj5dx8ngNMmbcy6SbgXZBd8U,61450 +numpy/core/include/numpy/__ufunc_api.c,sha256=670Gcz-vhkF4taBDmktCpFRBrZ9CHJnPRx7ag7Z6HsI,1714 +numpy/core/include/numpy/__ufunc_api.h,sha256=0MBOl7dgO3ldqdDi-SdciEOuqGv1UNsmk7mp7tEy4AY,12456 +numpy/core/include/numpy/_dtype_api.h,sha256=4veCexGvx9KNWMIUuEUAVOfcsei9GqugohDY5ud16pA,16697 +numpy/core/include/numpy/_neighborhood_iterator_imp.h,sha256=s-Hw_l5WRwKtYvsiIghF0bg-mA_CgWnzFFOYVFJ-q4k,1857 +numpy/core/include/numpy/_numpyconfig.h,sha256=o0fV_jb-wgVtRxnVIWvUttiZafyrWYFm2ab9Uixz1Cw,855 +numpy/core/include/numpy/arrayobject.h,sha256=-BlWQ7kfVbzCqzHn0qaeMe0_08AbwliuG98XWG57lT8,282 +numpy/core/include/numpy/arrayscalars.h,sha256=C3vDRndZTZRbppiDyV5jp8sV3dRKsrwBIZcNlh9gSTA,3944 +numpy/core/include/numpy/experimental_dtype_api.h,sha256=tlehD5r_pYhHbGzIrUea6vtOgf6IQ8Txblnhx7455h8,15532 +numpy/core/include/numpy/halffloat.h,sha256=TRZfXgipa-dFppX2uNgkrjrPli-1BfJtadWjAembJ4s,1959 +numpy/core/include/numpy/ndarrayobject.h,sha256=PhY4NjRZDoU5Zbc8MW0swPEm81hwgWZ63gAU93bLVVI,10183 +numpy/core/include/numpy/ndarraytypes.h,sha256=EjWXv-J8C5JET4AlIbJRdctycL7-dyJZcnoWgnlCPc8,68009 +numpy/core/include/numpy/noprefix.h,sha256=d83l1QpCCVqMV2k29NMkL3Ld1qNjiC6hzOPWZAivEjQ,6830 +numpy/core/include/numpy/npy_1_7_deprecated_api.h,sha256=y0MJ8Qw7Bkt4H_4VxIzHzpkw5JqAdj5ECgtn08fZFrI,4327 +numpy/core/include/numpy/npy_3kcompat.h,sha256=SvN9yRA3i02O4JFMXxZz0Uq_vJ5ZpvC-pC2sfF56A5I,15883 +numpy/core/include/numpy/npy_common.h,sha256=apWBsCJeP8P5T0exgzhFcGohbASsUF8vtFdS2jc1VfU,37746 +numpy/core/include/numpy/npy_cpu.h,sha256=pcVRtj-Y6120C5kWB1VAiAjZoxkTPDEg0gGm5IAt3jM,4629 +numpy/core/include/numpy/npy_endian.h,sha256=we7X9fPeWzNpo_YTh09MPGDwdE0Rw_WDM4c9y4nBj5I,2786 +numpy/core/include/numpy/npy_interrupt.h,sha256=DQZIxi6FycLXD8drdHn2SSmLoRhIpo6osvPv13vowUA,1948 +numpy/core/include/numpy/npy_math.h,sha256=SbKRoc7O3gVuDl7HOZjk424O049I0zn-7i9GwBwNmmk,18945 +numpy/core/include/numpy/npy_no_deprecated_api.h,sha256=0yZrJcQEJ6MCHJInQk5TP9_qZ4t7EfBuoLOJ34IlJd4,678 +numpy/core/include/numpy/npy_os.h,sha256=hlQsg_7-RkvS3s8OM8KXy99xxyJbCm-W1AYVcdnO1cw,1256 +numpy/core/include/numpy/numpyconfig.h,sha256=Nr59kE3cXmen6y0UymIBaU7F1BSIuPwgKZ4gdV5Q5JU,5308 +numpy/core/include/numpy/old_defines.h,sha256=xuYQDDlMywu0Zsqm57hkgGwLsOFx6IvxzN2eiNF-gJY,6405 +numpy/core/include/numpy/random/LICENSE.txt,sha256=-8U59H0M-DvGE3gID7hz1cFGMBJsrL_nVANcOSbapew,1018 +numpy/core/include/numpy/random/bitgen.h,sha256=49AwKOR552r-NkhuSOF1usb_URiMSRMvD22JF5pKIng,488 +numpy/core/include/numpy/random/distributions.h,sha256=W5tOyETd0m1W0GdaZ5dJP8fKlBtsTpG23V2Zlmrlqpg,9861 +numpy/core/include/numpy/random/libdivide.h,sha256=ew9MNhPQd1LsCZiWiFmj9IZ7yOnA3HKOXffDeR9X1jw,80138 +numpy/core/include/numpy/ufuncobject.h,sha256=Xmnny_ulZo9VwxkfkXF-1HCTKDavIp9PV_H7XWhi0Z8,12070 +numpy/core/include/numpy/utils.h,sha256=wMNomSH3Dfj0q78PrjLVtFtN-FPo7UJ4o0ifCUO-6Es,1185 +numpy/core/lib/libnpymath.a,sha256=mb8EluEp8SLpEeCTQJ0VshL-CqeZfWxSbS5ItM-9POc,93960 +numpy/core/lib/npy-pkg-config/mlib.ini,sha256=_LsWV1eStNqwhdiYPa2538GL46dnfVwT4MrI1zbsoFw,147 +numpy/core/lib/npy-pkg-config/npymath.ini,sha256=kamUNrYKAmXqQa8BcNv7D5sLqHh6bnChM0_5rZCsTfY,360 +numpy/core/memmap.py,sha256=yWBJLeVClHsD8BYusnf9bdqypOMPrj3_zoO_lQ2zVMc,11771 +numpy/core/memmap.pyi,sha256=sxIQ7T5hPLG-RBNndAc8JPvrsKEX1amBSH2HGg48Obo,55 +numpy/core/multiarray.py,sha256=zXaWf_DSkFEWjUQqVRCGeevwsI6kjQ3x6_MUwA1Y8fk,56097 +numpy/core/multiarray.pyi,sha256=_0X4W90U5ZiKt2n-9OscK-pcQyV6oGK-8jwGy5k1qxA,24768 +numpy/core/numeric.py,sha256=DgajaCDXiiQR-zuW_rrx_QhApSsa5k5FONK3Uk9mfTs,77014 +numpy/core/numeric.pyi,sha256=oVQkI4ABayFl_ZzCiGH4DxfYASL-3aETi-3B93THnEQ,14315 +numpy/core/numerictypes.py,sha256=qIf9v1OpNjjVQzXnKpD-3V01y5Bj9huw5F-U5Wa4glc,18098 +numpy/core/numerictypes.pyi,sha256=dEqtq9MLrGaqqeAF1sdXBgnEwDWOzlK02A6MTg1PS5g,3267 +numpy/core/overrides.py,sha256=YUZFS8RCBvOJ27sH-jDRcyMjOCn9VigMyuQY4J21JBI,7093 +numpy/core/records.py,sha256=4mpIjUp2XtZxY5cD2S8mgfn8GCzQGGrrkqLBqAJwM-Q,37533 +numpy/core/records.pyi,sha256=uYwE6cAoGKgN6U4ryfGZx_3m-3sY006jytjWLrDRRy0,5692 +numpy/core/shape_base.py,sha256=RPMKxA7_FCAgg_CruExl0LehnczSTFaxA6hrcfrUzns,29743 +numpy/core/shape_base.pyi,sha256=Ilb4joJmbjkIZLzKww7NJeaxg2FP3AfFib3HtfOsrC0,2774 +numpy/core/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/core/tests/_locales.py,sha256=S4x5soqF0oxpBYOE8J9Iky72O9J25IiZ8349m93pWC4,2206 +numpy/core/tests/data/astype_copy.pkl,sha256=lWSzCcvzRB_wpuRGj92spGIw-rNPFcd9hwJaRVvfWdk,716 +numpy/core/tests/data/generate_umath_validation_data.cpp,sha256=fyhQPNhIX9hzjeXujn6mhi1MVc133zELSV_hlSQ7BQU,5842 +numpy/core/tests/data/numpy_2_0_array.pkl,sha256=Vh02tdyCypa8Nb4QzdVhnDAiXEO2WQrcwcvOdDDFF5w,718 +numpy/core/tests/data/recarray_from_file.fits,sha256=NA0kliz31FlLnYxv3ppzeruONqNYkuEvts5wzXEeIc4,8640 +numpy/core/tests/data/umath-validation-set-README.txt,sha256=pxWwOaGGahaRd-AlAidDfocLyrAiDp0whf5hC7hYwqM,967 +numpy/core/tests/data/umath-validation-set-arccos.csv,sha256=W_aL99bjzVjlVyd5omfDUORag8jHzx6uctedPVZgOHQ,61365 +numpy/core/tests/data/umath-validation-set-arccosh.csv,sha256=Uko_d0kDXr1YlN-6Ii-fQQxUvbXAhRfC7Un4gJ23GJk,61365 +numpy/core/tests/data/umath-validation-set-arcsin.csv,sha256=15Aenze4WD2a2dF2aOBXpv9B7u3wwAeUVJdEm4TjOkQ,61339 +numpy/core/tests/data/umath-validation-set-arcsinh.csv,sha256=uDwx4PStpfV21IaPF8pmzQpul6i72g7zDwlfcynWaVQ,60289 +numpy/core/tests/data/umath-validation-set-arctan.csv,sha256=mw5tYze_BMs6ugGEZfg5mcXoInGYdn7fvSCYSUi9Bqw,60305 +numpy/core/tests/data/umath-validation-set-arctanh.csv,sha256=95l4Uu5RmZajljabfqlv5U34RVrifCMhhkop6iLeNBo,61339 +numpy/core/tests/data/umath-validation-set-cbrt.csv,sha256=v855MTZih-fZp_GuEDst2qaIsxU4a7vlAbeIJy2xKpc,60846 +numpy/core/tests/data/umath-validation-set-cos.csv,sha256=0PNnDqKkokZ7ERVDgbes8KNZc-ISJrZUlVZc5LkW18E,59122 +numpy/core/tests/data/umath-validation-set-cosh.csv,sha256=FGCNeUSUTAeASsb_j18iRSsCxXLxmzF-_C7tq1elVrQ,60869 +numpy/core/tests/data/umath-validation-set-exp.csv,sha256=BKg1_cyrKD2GXYMX_EB0DnXua8DI2O1KWODXf_BRhrk,17491 +numpy/core/tests/data/umath-validation-set-exp2.csv,sha256=f1b05MRXPOXihC9M-yi52udKBzVXalhbTuIcqoDAk-g,58624 +numpy/core/tests/data/umath-validation-set-expm1.csv,sha256=_ghc1xiUECNsBGrKCFUAy2lvu01_lkpeYJN0zDtCYWk,60299 +numpy/core/tests/data/umath-validation-set-log.csv,sha256=z9ej1ykKUoMRqYMUIJENWXbYi_A_x_RKs7K_GuXZJus,11692 +numpy/core/tests/data/umath-validation-set-log10.csv,sha256=RJgpruL16FVPgUT3-3xW4eppS_tn6o5yEW79KnITn48,68922 +numpy/core/tests/data/umath-validation-set-log1p.csv,sha256=IZZI-hi55HGCOvBat3vSBVha_8Nt-5alf2fqz6QeTG0,60303 +numpy/core/tests/data/umath-validation-set-log2.csv,sha256=HL2rOCsrEi378rNrbsXHPqlWlEGkXQq8R4e63YeTksU,68917 +numpy/core/tests/data/umath-validation-set-sin.csv,sha256=8PUjnQ_YfmxFb42XJrvpvmkeSpEOlEXSmNvIK4VgfAM,58611 +numpy/core/tests/data/umath-validation-set-sinh.csv,sha256=CYiibE8aX7MQnBatl__5k_PWc_9vHUifwS-sFZzzKk0,60293 +numpy/core/tests/data/umath-validation-set-tan.csv,sha256=Oq7gxMvblRVBrQ23kMxc8iT0bHnCWKg9EE4ZqzbJbOA,60299 +numpy/core/tests/data/umath-validation-set-tanh.csv,sha256=iolZF_MOyWRgYSa-SsD4df5mnyFK18zrICI740SWoTc,60299 +numpy/core/tests/examples/cython/checks.pyx,sha256=rKAhPSGHJ9oPK9Q_85YoUQyRTftEP1jcYOR5lSPB6oQ,662 +numpy/core/tests/examples/cython/meson.build,sha256=Qk4Q6OkpZ0xsLUkcGQVVrYkzb0ozoyL6YlSZ8_5tH1I,1088 +numpy/core/tests/examples/cython/setup.py,sha256=aAR-TvQabUabnCzuB6UdWdmRXaaPfIG7MzTIfMF-0tk,496 +numpy/core/tests/examples/limited_api/limited_api.c,sha256=mncE8TjjXmYpkwli433G0jB2zGQO_5NqWmGKdzRJZug,344 +numpy/core/tests/examples/limited_api/setup.py,sha256=p2w7F1ardi_GRXSrnNIR8W1oeH_pgmw_1P2wS0A2I6M,435 +numpy/core/tests/test__exceptions.py,sha256=QqxQSLXboPXEVwHz-TyE2JeIl_TC-rPugzfo25nbcns,2846 +numpy/core/tests/test_abc.py,sha256=FfgYA_HjYAi8XWGK_oOh6Zw86chB_KG_XoW_7ZlFp4c,2220 +numpy/core/tests/test_api.py,sha256=UMc7SvczAQ5ngHxE-NoXVvNpVzYRrn8oMwFNta1yMS0,22995 +numpy/core/tests/test_argparse.py,sha256=C0zBbwQ9xzzymXe_hHpWnnWQPwOi2ZdQB78gBAgJHvU,1969 +numpy/core/tests/test_array_coercion.py,sha256=zY4Pjlt4QZ0w71WxWGLHcrPnnhEF51yXYVLg5HMIy5c,34379 +numpy/core/tests/test_array_interface.py,sha256=8tGgj1Nzi76H_WF5GULkxqWL7Yu_Xf0lvTJZOwOBKsI,7774 +numpy/core/tests/test_arraymethod.py,sha256=VpjDYTmoMDTZcY7CsGzinBh0R_OICuwOykWCbmCRQZU,3244 +numpy/core/tests/test_arrayprint.py,sha256=cKaIoD9ZvsjJH0PHwZyOxmcRcBt1kN1WfFneqVqs0b8,40462 +numpy/core/tests/test_casting_floatingpoint_errors.py,sha256=W3Fgk0oKtXFv684fEZ7POwj6DHTYK0Jj_oGRLZ8UdyA,5063 +numpy/core/tests/test_casting_unittests.py,sha256=9-vkR0oXczQz8ED8DxGVPmalC8IZXe2jKgOCMGr8hIg,34298 +numpy/core/tests/test_conversion_utils.py,sha256=jNhbNNI-T8qtQnsIMEax7KFN30kjh0ICntLMwTyxJ5Q,6559 +numpy/core/tests/test_cpu_dispatcher.py,sha256=v_SlhUpENuoe7QYXizzYITLGXa7WfZ7jqcqmbSBg7JU,1542 +numpy/core/tests/test_cpu_features.py,sha256=mieGx7dxXFiyTYatbcCCjIjR67Un2hVcbJx4GEf2yFo,14892 +numpy/core/tests/test_custom_dtypes.py,sha256=JogRmttDLwfQ3PTbewEnGLKco9zV2Nu3yIfrMeCsx_I,9401 +numpy/core/tests/test_cython.py,sha256=t5-h4XSIFNLyw_9BIAQDYl8_80t_pH0SCfEa1Vf_3aI,3755 +numpy/core/tests/test_datetime.py,sha256=2vAGbrCQmsrWNXCVXOMZqUGZn2c-cQT-eZ1wTprYbcM,116211 +numpy/core/tests/test_defchararray.py,sha256=F88HUkByEP4H6cJ_ITvIe0a_T1BH2JOdRysMCu1XIn0,24997 +numpy/core/tests/test_deprecations.py,sha256=w2lhHb-W8hh7RoE_0Ftg8thpG86jvbFAJgior22DY2Q,31076 +numpy/core/tests/test_dlpack.py,sha256=cDlwFmTombb2rDeB8RHEAJ4eVMUiDbw8Oz5Jo1NQwk0,3522 +numpy/core/tests/test_dtype.py,sha256=J09pJF59v7UO6iNuJFISKP2DLPgdkQ_df5OAMDRLikU,75702 +numpy/core/tests/test_einsum.py,sha256=QzQAPIC-IjTV3Dxz97hBnvLBCmF8kpsBTBckThhgRjQ,53712 +numpy/core/tests/test_errstate.py,sha256=U3GT9I058jkF725mx4GdWUr9RoceCkGDV7Go79VA4wY,2219 +numpy/core/tests/test_extint128.py,sha256=gCZfAwPOb-F1TLsEEeDI0amQYwHk-60-OXi0ccZrrZ8,5643 +numpy/core/tests/test_function_base.py,sha256=Ibs6-WXZE5hsRx4VCnX-cZOWYKU-5PFXjouwAQzgnqQ,15595 +numpy/core/tests/test_getlimits.py,sha256=apdxr0zKkxaVHIUpLrqAvO39q54JKN14sV4xSbK2Ifs,6718 +numpy/core/tests/test_half.py,sha256=VYPyap9GYOWZuphsfFofcIRl-oa5Ufrtv83OTp6azdU,24593 +numpy/core/tests/test_hashtable.py,sha256=ZV8HL8NkDnoQZfnje7BP0fyIp4fSFqjKsQc40PaTggc,1011 +numpy/core/tests/test_indexerrors.py,sha256=kN9xLl6FVTzmI7fumn_cuZ3k0omXnTetgtCnPY44cvw,5130 +numpy/core/tests/test_indexing.py,sha256=x0ojWuhOwWD5MZuiJ9Ncim3CgkwI-GldWxrSCmjmFJM,54314 +numpy/core/tests/test_item_selection.py,sha256=kI30kiX8mIrZYPn0jw3lGGw1ruZF4PpE9zw-aai9EPA,6458 +numpy/core/tests/test_limited_api.py,sha256=5yO0nGmCKZ9b3S66QP7vY-HIgAoyOtHZmp8mvzKuOHI,1172 +numpy/core/tests/test_longdouble.py,sha256=jO8YMm_Hsz-XPKbmv6iMcOdHgTlIFkKTwAtxpy3Q1pE,13905 +numpy/core/tests/test_machar.py,sha256=_5_TDUVtAJvJI5jBfEFKpCZtAfKCsCFt7tXlWSkWzzc,1067 +numpy/core/tests/test_mem_overlap.py,sha256=QJ0unWD_LOoAGAo4ra0IvYenj56IYUtiz1fEJEmTY9Q,29086 +numpy/core/tests/test_mem_policy.py,sha256=CXa10FQw2Qj6MqJuaC8Fm4slsoipKFjCIpYF6c5IIAU,16801 +numpy/core/tests/test_memmap.py,sha256=tZ5lJs_4ZFsJmg392ZQ33fX0m8tdfZ8ZtY9Lq41LNtk,7477 +numpy/core/tests/test_multiarray.py,sha256=GPv4IJR9dijNG-icUsQsX2tBD2RdP3EhUehY4cxvVQU,380106 +numpy/core/tests/test_nditer.py,sha256=nVQ00aNxPHqf4ZcFs3e9AVDK64TCqlO0TzfocTAACZQ,130818 +numpy/core/tests/test_nep50_promotions.py,sha256=2TwtFvj1LBpYTtdR6NFe1RAAGXIJltLqwpA1vhQCVY4,8840 +numpy/core/tests/test_numeric.py,sha256=ZGNW5NKgShEjZC_TcPOtTuRaTM_GbuM21u82D205UPs,137294 +numpy/core/tests/test_numerictypes.py,sha256=f_xMjZJnyDwlc6XCrd71b6x1_6dAWOv-kZ3-NEq37hU,21687 +numpy/core/tests/test_numpy_2_0_compat.py,sha256=kVCTAXska7Xi5w_TYduWhid0nlCqI6Nvmt-gDnYsuKI,1630 +numpy/core/tests/test_overrides.py,sha256=t0gOZOzu7pevE58HA-npFYJqnInHR-LLBklnzKJWHqo,26080 +numpy/core/tests/test_print.py,sha256=ErZAWd88b0ygSEoYpd0BL2tFjkerMtn1vZ7dWvaNqTc,6837 +numpy/core/tests/test_protocols.py,sha256=fEXE9K9s22oiVWkX92BY-g00-uXCK-HxjZhZxxYAKFc,1168 +numpy/core/tests/test_records.py,sha256=pluit5x6jkWoPEIrHXM13L3xZuuSSiaxoXFsOdkakCU,20269 +numpy/core/tests/test_regression.py,sha256=SJo9cPTVr2SNjhgtW7boUMyNQlXxygsZ5g0oyqC8Eks,91595 +numpy/core/tests/test_scalar_ctors.py,sha256=qDIZV-tBukwAxNDhUmGtH3CemDXlS3xd_q3L52touuA,6115 +numpy/core/tests/test_scalar_methods.py,sha256=Uj-zU0zzzKAjMBdpkzsWZ3nSFj5gJkUlqi_euhOYdnU,7541 +numpy/core/tests/test_scalarbuffer.py,sha256=FSL94hriWX1_uV6Z33wB3ZXUrpmmX2-x87kNjIxUeBk,5580 +numpy/core/tests/test_scalarinherit.py,sha256=fMInDGKsiH3IS_2ejZtIcmJZ0Ry8c7kVsHx7wp5XDoM,2368 +numpy/core/tests/test_scalarmath.py,sha256=XZj_m2I2TLktJdFD1SWj2XtV8hT26VIxasDz3cAFvgA,43247 +numpy/core/tests/test_scalarprint.py,sha256=1599W5X0tjGhBnSQjalXkg6AY8eHXnr6PMqs4vYZQqs,18771 +numpy/core/tests/test_shape_base.py,sha256=D9haeuUVx3x3pOLmFQ9vUz7iU4T2bFTsPoI8HgSncFU,29723 +numpy/core/tests/test_simd.py,sha256=-L1UhIn9Eu_euLwaSU7bPRfYpWWOTb43qovoJS7Ws7w,48696 +numpy/core/tests/test_simd_module.py,sha256=OSpYhH_3QDxItyQcaW6SjXW57k2m-weRwpYOnJjCqN0,3902 +numpy/core/tests/test_strings.py,sha256=A9t1B65lFrYRLXgDJSg3mMDAe_hypIPcTMVOdAYIbU0,3835 +numpy/core/tests/test_ufunc.py,sha256=5pS2x3LACHn8GogYYad8LRAjByK7Gg9xTD9ik3d0Fm0,124907 +numpy/core/tests/test_umath.py,sha256=huHpclJqkO32k7BTflRHj8nImzg3p6yyryeS9LyHKWU,186482 +numpy/core/tests/test_umath_accuracy.py,sha256=mFcVdzXhhD9mqhzLDJVZsWfCHbjbFQ6XeEl5G8l-PTc,3897 +numpy/core/tests/test_umath_complex.py,sha256=WvZZZWeijo52RiOfx-G83bxzQOp_IJ3i9fEnUDVukLQ,23247 +numpy/core/tests/test_unicode.py,sha256=hUXIwMmoq89y_KXWzuXVyQaXvRwGjfY4TvKJsCbygEI,12775 +numpy/core/umath.py,sha256=JbT_SxnZ_3MEmjOI9UtX3CcAzX5Q-4RDlnnhDAEJ5Vo,2040 +numpy/core/umath_tests.py,sha256=TIzaDfrEHHgSc2J5kxFEibq8MOPhwSuyOZOUBsZNVSM,389 +numpy/ctypeslib.py,sha256=Po4XCWfxhwFQ1Q8x8DeayGiMCJLxREaCDkVyeladxBU,17247 +numpy/ctypeslib.pyi,sha256=A9te473aRO920iDVuyKypeVIQp-ueZK6EiI-qLSwJNg,7972 +numpy/doc/__init__.py,sha256=OYmE-F6x0CD05PCDY2MiW1HLlwB6i9vhDpk-a3r4lHY,508 +numpy/doc/constants.py,sha256=PlXoj7b4A8Aa9nADbg83uzTBRJaX8dvJmEdbn4FDPPo,9155 +numpy/doc/ufuncs.py,sha256=i1alLg19mNyCFZ2LYSOZGm--RsRN1x63U_UYU-N3x60,5357 +numpy/dtypes.py,sha256=BuBztrPQRasUmVZhXr2_NgJujdUTNhNwd59pZZHk3lA,2229 +numpy/dtypes.pyi,sha256=tIHniAYP7ALg2iT7NgSXO67jvE-zRlDod3MazEmD4M8,1315 +numpy/exceptions.py,sha256=7j7tv8cwXGZYgldyMisGmnAxAl2s4YU0vexME81yYlA,7339 +numpy/exceptions.pyi,sha256=KsZqWNvyPUEXUGR9EhZCUQF2f9EVSpBRlJUlGqRT02k,600 +numpy/f2py/__init__.py,sha256=m-ty_WiJZ4GVfV5--kJ3MFJaLXestz5Eo-4H0FPscK4,5565 +numpy/f2py/__init__.pyi,sha256=eA7uYXZr0p0aaz5rBW-EypLx9RchrvqDYtSnkEJQsYw,1087 +numpy/f2py/__main__.py,sha256=6i2jVH2fPriV1aocTY_dUFvWK18qa-zjpnISA-OpF3w,130 +numpy/f2py/__version__.py,sha256=7HHdjR82FCBmftwMRyrlhcEj-8mGQb6oCH-wlUPH4Nw,34 +numpy/f2py/_backends/__init__.py,sha256=7_bA7c_xDpLc4_8vPfH32-Lxn9fcUTgjQ25srdvwvAM,299 +numpy/f2py/_backends/_backend.py,sha256=GKb9-UaFszT045vUgVukPs1n97iyyjqahrWKxLOKNYo,1187 +numpy/f2py/_backends/_distutils.py,sha256=pxh2YURFYYSykIOvBFwVvhoNX1oSk-c30IPPhzlko-0,2383 +numpy/f2py/_backends/_meson.py,sha256=gi-nbnPFDC38sumfAjg-Q5FPu6nNkyQXTjEuVf9W9Cc,6916 +numpy/f2py/_backends/meson.build.template,sha256=oTPNMAQzS4CJ_lfEzYv-oBeJTtQuThUYVN5R6ROWpNU,1579 +numpy/f2py/_isocbind.py,sha256=zaBgpfPNRmxVG3doUIlbZIiyB990MsXiwDabrSj9HnQ,2360 +numpy/f2py/_src_pyf.py,sha256=4t6TN4ZKWciC4f1z6fwaGrpIGhHKRiwHfcrNj4FIzCg,7654 +numpy/f2py/auxfuncs.py,sha256=dNs4b2KDIcG4M1hPBvD09-Vh7CDzlPIrFscOdvL3p1o,26539 +numpy/f2py/capi_maps.py,sha256=ENjYyeZ3CCJcLwJJgmKOSYrD1KPuhpwauXqeizdV55o,30563 +numpy/f2py/cb_rules.py,sha256=5TuHbJWGjsF6yVNzKuV2tAnwdLyhcWlmdsjYlDOZOv4,24992 +numpy/f2py/cfuncs.py,sha256=KJyW7mdjmFSmxssfeegGJs5NZyF3mZMgNvOxN9-vYHQ,51913 +numpy/f2py/common_rules.py,sha256=gHB76WypbkVmhaD_RWhy8Od4zDTgj8cbDOdUdIp6PIQ,5131 +numpy/f2py/crackfortran.py,sha256=ErLdkWP8MxeyW5vVPGXwyvrxZAwymlvIBC0th2rvK74,148553 +numpy/f2py/diagnose.py,sha256=0SRXBE2hJgKJN_Rf4Zn00oKXC_Tka3efPWM47zg6BoY,5197 +numpy/f2py/f2py2e.py,sha256=5t093ZQ4xs0_0UbyaYVd2yA2EVOaOAcuU29JI-IU2Ag,27717 +numpy/f2py/f90mod_rules.py,sha256=otm3_dmVIna0eBVHLu_693s3a_82lU3pqeqDacWI37s,9594 +numpy/f2py/func2subr.py,sha256=6d2R5awuHRT4xzgfUfwS7JHTqhhAieSXcENlssD_2c4,10298 +numpy/f2py/rules.py,sha256=B4FxSYEfZ_1j_z9GulQNZ1BNrPrUvlU3ybxwTkrIxjI,62727 +numpy/f2py/setup.cfg,sha256=Fpn4sjqTl5OT5sp8haqKIRnUcTPZNM6MIvUJBU7BIhg,48 +numpy/f2py/setup.py,sha256=MmAVspT8DDTqDuL8ZJhxK62g0lcso4vqI6QNQ9CsfoQ,2422 +numpy/f2py/src/fortranobject.c,sha256=g4BKDO1_9pCu6hithKXD2oH_Mt-HH1NTnP6leCqJrzc,46017 +numpy/f2py/src/fortranobject.h,sha256=neMKotYWbHvrhW9KXz4QzQ8fzPkiQXLHHjy82vLSeog,5835 +numpy/f2py/symbolic.py,sha256=jWBoAwECCxRdWczR9r7O6UERcYmH_GbdcAReNp7cmJY,53270 +numpy/f2py/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/f2py/tests/src/abstract_interface/foo.f90,sha256=JFU2w98cB_XNwfrqNtI0yDTmpEdxYO_UEl2pgI_rnt8,658 +numpy/f2py/tests/src/abstract_interface/gh18403_mod.f90,sha256=gvQJIzNtvacWE0dhysxn30-iUeI65Hpq7DiE9oRauz8,105 +numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c,sha256=Ff5wHYV9-OJnZuelfFWcjAibRvDkEIlbTVczTyv6TG8,7299 +numpy/f2py/tests/src/assumed_shape/.f2py_f2cmap,sha256=But9r9m4iL7EGq_haMW8IiQ4VivH0TgUozxX4pPvdpE,29 +numpy/f2py/tests/src/assumed_shape/foo_free.f90,sha256=oBwbGSlbr9MkFyhVO2aldjc01dr9GHrMrSiRQek8U64,460 +numpy/f2py/tests/src/assumed_shape/foo_mod.f90,sha256=rfzw3QdI-eaDSl-hslCgGpd5tHftJOVhXvb21Y9Gf6M,499 +numpy/f2py/tests/src/assumed_shape/foo_use.f90,sha256=rmT9k4jP9Ru1PLcGqepw9Jc6P9XNXM0axY7o4hi9lUw,269 +numpy/f2py/tests/src/assumed_shape/precision.f90,sha256=r08JeTVmTTExA-hYZ6HzaxVwBn1GMbPAuuwBhBDtJUk,130 +numpy/f2py/tests/src/block_docstring/foo.f,sha256=y7lPCPu7_Fhs_Tf2hfdpDQo1bhtvNSKRaZAOpM_l3dg,97 +numpy/f2py/tests/src/callback/foo.f,sha256=C1hjfpRCQWiOVVzIHqnsYcnLrqQcixrnHCn8hd9GhVk,1254 +numpy/f2py/tests/src/callback/gh17797.f90,sha256=_Nrl0a2HgUbtymGU0twaJ--7rMa1Uco2A3swbWvHoMo,148 +numpy/f2py/tests/src/callback/gh18335.f90,sha256=NraOyKIXyvv_Y-3xGnmTjtNjW2Znsnlk8AViI8zfovc,506 +numpy/f2py/tests/src/callback/gh25211.f,sha256=a2sxlQhtDVbYn8KOKHUYqwc-aCFt7sDPSnJsXFG35uI,179 +numpy/f2py/tests/src/callback/gh25211.pyf,sha256=FWxo0JWQlw519BpZV8PoYeI_FZ_K6C-3Wk6gLrfBPlw,447 +numpy/f2py/tests/src/cli/gh_22819.pyf,sha256=5rvOfCv-wSosB354LC9pExJmMoSHnbGZGl_rtA2fogA,142 +numpy/f2py/tests/src/cli/hi77.f,sha256=ttyI6vAP3qLnDqy82V04XmoqrXNM6uhMvvLri2p0dq0,71 +numpy/f2py/tests/src/cli/hiworld.f90,sha256=QWOLPrTxYQu1yrEtyQMbM0fE9M2RmXe7c185KnD5x3o,51 +numpy/f2py/tests/src/common/block.f,sha256=GQ0Pd-VMX3H3a-__f2SuosSdwNXHpBqoGnQDjf8aG9g,224 +numpy/f2py/tests/src/common/gh19161.f90,sha256=BUejyhqpNVfHZHQ-QC7o7ZSo7lQ6YHyX08lSmQqs6YM,193 +numpy/f2py/tests/src/crackfortran/accesstype.f90,sha256=-5Din7YlY1TU7tUHD2p-_DSTxGBpDsWYNeT9WOwGhno,208 +numpy/f2py/tests/src/crackfortran/data_common.f,sha256=ZSUAh3uhn9CCF-cYqK5TNmosBGPfsuHBIEfudgysun4,193 +numpy/f2py/tests/src/crackfortran/data_multiplier.f,sha256=jYrJKZWF_59JF9EMOSALUjn0UupWvp1teuGpcL5s1Sc,197 +numpy/f2py/tests/src/crackfortran/data_stmts.f90,sha256=19YO7OGj0IksyBlmMLZGRBQLjoE3erfkR4tFvhznvvE,693 +numpy/f2py/tests/src/crackfortran/data_with_comments.f,sha256=hoyXw330VHh8duMVmAQZjr1lgLVF4zFCIuEaUIrupv0,175 +numpy/f2py/tests/src/crackfortran/foo_deps.f90,sha256=CaH7mnWTG7FcnJe2vXN_0zDbMadw6NCqK-JJ2HmDjK8,128 +numpy/f2py/tests/src/crackfortran/gh15035.f,sha256=jJly1AzF5L9VxbVQ0vr-sf4LaUo4eQzJguhuemFxnvg,375 +numpy/f2py/tests/src/crackfortran/gh17859.f,sha256=7K5dtOXGuBDAENPNCt-tAGJqTfNKz5OsqVSk16_e7Es,340 +numpy/f2py/tests/src/crackfortran/gh22648.pyf,sha256=qZHPRNQljIeYNwbqPLxREnOrSdVV14f3fnaHqB1M7c0,241 +numpy/f2py/tests/src/crackfortran/gh23533.f,sha256=w3tr_KcY3s7oSWGDmjfMHv5h0RYVGUpyXquNdNFOJQg,126 +numpy/f2py/tests/src/crackfortran/gh23598.f90,sha256=41W6Ire-5wjJTTg6oAo7O1WZfd1Ug9vvNtNgHS5MhEU,101 +numpy/f2py/tests/src/crackfortran/gh23598Warn.f90,sha256=1v-hMCT_K7prhhamoM20nMU9zILam84Hr-imck_dYYk,205 +numpy/f2py/tests/src/crackfortran/gh23879.f90,sha256=LWDJTYR3t9h1IsrKC8dVXZlBfWX7clLeU006X6Ow8oI,332 +numpy/f2py/tests/src/crackfortran/gh2848.f90,sha256=gPNasx98SIf7Z9ibk_DHiGKCvl7ERtsfoGXiFDT7FbM,282 +numpy/f2py/tests/src/crackfortran/operators.f90,sha256=-Fc-qjW1wBr3Dkvdd5dMTrt0hnjnV-1AYo-NFWcwFSo,1184 +numpy/f2py/tests/src/crackfortran/privatemod.f90,sha256=7bubZGMIn7iD31wDkjF1TlXCUM7naCIK69M9d0e3y-U,174 +numpy/f2py/tests/src/crackfortran/publicmod.f90,sha256=Pnwyf56Qd6W3FUH-ZMgnXEYkb7gn18ptNTdwmGan0Jo,167 +numpy/f2py/tests/src/crackfortran/pubprivmod.f90,sha256=eYpJwBYLKGOxVbKgEqfny1znib-b7uYhxcRXIf7uwXg,165 +numpy/f2py/tests/src/crackfortran/unicode_comment.f90,sha256=aINLh6GlfTwFewxvDoqnMqwuCNb4XAqi5Nj5vXguXYs,98 +numpy/f2py/tests/src/f2cmap/.f2py_f2cmap,sha256=iUOtfHd3OuT1Rz2-yiSgt4uPKGvCt5AzQ1iygJt_yjg,82 +numpy/f2py/tests/src/f2cmap/isoFortranEnvMap.f90,sha256=iJCD8a8MUTmuPuedbcmxW54Nr4alYuLhksBe1sHS4K0,298 +numpy/f2py/tests/src/isocintrin/isoCtests.f90,sha256=jcw-fzrFh0w5U66uJYfeUW4gv94L5MnWQ_NpsV9y0oI,998 +numpy/f2py/tests/src/kind/foo.f90,sha256=zIHpw1KdkWbTzbXb73hPbCg4N2Htj3XL8DIwM7seXpo,347 +numpy/f2py/tests/src/mixed/foo.f,sha256=90zmbSHloY1XQYcPb8B5d9bv9mCZx8Z8AMTtgDwJDz8,85 +numpy/f2py/tests/src/mixed/foo_fixed.f90,sha256=pxKuPzxF3Kn5khyFq9ayCsQiolxB3SaNtcWaK5j6Rv4,179 +numpy/f2py/tests/src/mixed/foo_free.f90,sha256=fIQ71wrBc00JUAVUj_r3QF9SdeNniBiMw6Ly7CGgPWU,139 +numpy/f2py/tests/src/module_data/mod.mod,sha256=EkjrU7NTZrOH68yKrz6C_eyJMSFSxGgC2yMQT9Zscek,412 +numpy/f2py/tests/src/module_data/module_data_docstring.f90,sha256=tDZ3fUlazLL8ThJm3VwNGJ75QIlLcW70NnMFv-JA4W0,224 +numpy/f2py/tests/src/negative_bounds/issue_20853.f90,sha256=fdOPhRi7ipygwYCXcda7p_dlrws5Hd2GlpF9EZ-qnck,157 +numpy/f2py/tests/src/parameter/constant_both.f90,sha256=-bBf2eqHb-uFxgo6Q7iAtVUUQzrGFqzhHDNaxwSICfQ,1939 +numpy/f2py/tests/src/parameter/constant_compound.f90,sha256=re7pfzcuaquiOia53UT7qNNrTYu2euGKOF4IhoLmT6g,469 +numpy/f2py/tests/src/parameter/constant_integer.f90,sha256=nEmMLitKoSAG7gBBEQLWumogN-KS3DBZOAZJWcSDnFw,612 +numpy/f2py/tests/src/parameter/constant_non_compound.f90,sha256=IcxESVLKJUZ1k9uYKoSb8Hfm9-O_4rVnlkiUU2diy8Q,609 +numpy/f2py/tests/src/parameter/constant_real.f90,sha256=quNbDsM1Ts2rN4WtPO67S9Xi_8l2cXabWRO00CPQSSQ,610 +numpy/f2py/tests/src/quoted_character/foo.f,sha256=WjC9D9171fe2f7rkUAZUvik9bkIf9adByfRGzh6V0cM,482 +numpy/f2py/tests/src/regression/gh25337/data.f90,sha256=9Uz8CHB9i3_mjC3cTOmkTgPAF5tWSwYacG3MUrU-SY0,180 +numpy/f2py/tests/src/regression/gh25337/use_data.f90,sha256=WATiDGAoCKnGgMzm_iMgmfVU0UKOQlk5Fm0iXCmPAkE,179 +numpy/f2py/tests/src/regression/inout.f90,sha256=CpHpgMrf0bqA1W3Ozo3vInDz0RP904S7LkpdAH6ODck,277 +numpy/f2py/tests/src/return_character/foo77.f,sha256=WzDNF3d_hUDSSZjtxd3DtE-bSx1ilOMEviGyYHbcFgM,980 +numpy/f2py/tests/src/return_character/foo90.f90,sha256=ULcETDEt7gXHRzmsMhPsGG4o3lGrcx-FEFaJsPGFKyA,1248 +numpy/f2py/tests/src/return_complex/foo77.f,sha256=8ECRJkfX82oFvGWKbIrCvKjf5QQQClx4sSEvsbkB6A8,973 +numpy/f2py/tests/src/return_complex/foo90.f90,sha256=c1BnrtWwL2dkrTr7wvlEqNDg59SeNMo3gyJuGdRwcDw,1238 +numpy/f2py/tests/src/return_integer/foo77.f,sha256=_8k1evlzBwvgZ047ofpdcbwKdF8Bm3eQ7VYl2Y8b5kA,1178 +numpy/f2py/tests/src/return_integer/foo90.f90,sha256=bzxbYtofivGRYH35Ang9ScnbNsVERN8-6ub5-eI-LGQ,1531 +numpy/f2py/tests/src/return_logical/foo77.f,sha256=FxiF_X0HkyXHzJM2rLyTubZJu4JB-ObLnVqfZwAQFl8,1188 +numpy/f2py/tests/src/return_logical/foo90.f90,sha256=9KmCe7yJYpi4ftkKOM3BCDnPOdBPTbUNrKxY3p37O14,1531 +numpy/f2py/tests/src/return_real/foo77.f,sha256=ZTrzb6oDrIDPlrVWP3Bmtkbz3ffHaaSQoXkfTGtCuFE,933 +numpy/f2py/tests/src/return_real/foo90.f90,sha256=gZuH5lj2lG6gqHlH766KQ3J4-Ero-G4WpOOo2MG3ohU,1194 +numpy/f2py/tests/src/size/foo.f90,sha256=IlFAQazwBRr3zyT7v36-tV0-fXtB1d7WFp6S1JVMstg,815 +numpy/f2py/tests/src/string/char.f90,sha256=ihr_BH9lY7eXcQpHHDQhFoKcbu7VMOX5QP2Tlr7xlaM,618 +numpy/f2py/tests/src/string/fixed_string.f90,sha256=5n6IkuASFKgYICXY9foCVoqndfAY0AQZFEK8L8ARBGM,695 +numpy/f2py/tests/src/string/gh24008.f,sha256=UA8Pr-_yplfOFmc6m4v9ryFQ8W9OulaglulefkFWD68,217 +numpy/f2py/tests/src/string/gh24662.f90,sha256=-Tp9Kd1avvM7AIr8ZukFA9RVr-wusziAnE8AvG9QQI4,197 +numpy/f2py/tests/src/string/gh25286.f90,sha256=2EpxvC-0_dA58MBfGQcLyHzpZgKcMf_W9c73C_Mqnok,304 +numpy/f2py/tests/src/string/gh25286.pyf,sha256=GjgWKh1fHNdPGRiX5ek60i1XSeZsfFalydWqjISPVV8,381 +numpy/f2py/tests/src/string/gh25286_bc.pyf,sha256=6Y9zU66NfcGhTXlFOdFjCSMSwKXpq5ZfAe3FwpkAsm4,384 +numpy/f2py/tests/src/string/scalar_string.f90,sha256=ACxV2i6iPDk-a6L_Bs4jryVKYJMEGUTitEIYTjbJes4,176 +numpy/f2py/tests/src/string/string.f,sha256=shr3fLVZaa6SyUJFYIF1OZuhff8v5lCwsVNBU2B-3pk,248 +numpy/f2py/tests/src/value_attrspec/gh21665.f90,sha256=JC0FfVXsnB2lZHb-nGbySnxv_9VHAyD0mKaLDowczFU,190 +numpy/f2py/tests/test_abstract_interface.py,sha256=C8-ly0_TqkmpQNZmwPHwo2IV2MBH0jQEjAhpqHrg8Y4,832 +numpy/f2py/tests/test_array_from_pyobj.py,sha256=Txff89VUeEhWqUCRVybIqsqH4YQvpk4Uyjmh_XjyMi0,24049 +numpy/f2py/tests/test_assumed_shape.py,sha256=FeaqtrWyBf5uyArcmI0D2e_f763aSMpgU3QmdDXe-tA,1466 +numpy/f2py/tests/test_block_docstring.py,sha256=SEpuq73T9oVtHhRVilFf1xF7nb683d4-Kv7V0kfL4AA,564 +numpy/f2py/tests/test_callback.py,sha256=cReSlVjgnoT74wmtNn-oEIZiJUTfRX7ljjlqJi716IQ,6494 +numpy/f2py/tests/test_character.py,sha256=3ugjM1liymMRbY8wub1eiap-jdyNYVHxlNZBqNoRLe4,21868 +numpy/f2py/tests/test_common.py,sha256=m7TTSJt5zUZKJF-MQUeTtCyxW7YwRBSETINXGPFu8S4,896 +numpy/f2py/tests/test_compile_function.py,sha256=9d_FZ8P2wbIlQ2qPDRrsFqPb4nMH8tiWqYZN-P_shCs,4186 +numpy/f2py/tests/test_crackfortran.py,sha256=y1x3U-jlQWD5rmTXz1I2RlTz7LEfbI6qxCDkR5fzPwY,13441 +numpy/f2py/tests/test_data.py,sha256=HFcmPYbiveKa-swJ8x8XlRR9sM0ESB9FEN-txZnHTok,2876 +numpy/f2py/tests/test_docs.py,sha256=jqtuHE5ZjxP4D8Of3Fkzz36F8_0qKbeS040_m0ac4v4,1662 +numpy/f2py/tests/test_f2cmap.py,sha256=p-Sylbr3ctdKT3UQV9FzpCuYPH5U7Vyn8weXFAjiI9o,391 +numpy/f2py/tests/test_f2py2e.py,sha256=eoswH-daMEBlueoVpxXrDloahCpr0RLzHbr3zBHOsjk,25423 +numpy/f2py/tests/test_isoc.py,sha256=_nPTPxNEEagiKriZBeFNesOattIlHDzaNKmj35xxDBY,1406 +numpy/f2py/tests/test_kind.py,sha256=aOMQSBoD_dw49acKN25_abEvQBLI27DsnWIb9CNpSAE,1671 +numpy/f2py/tests/test_mixed.py,sha256=Ctuw-H7DxhPjSt7wZdJ2xffawIoEBCPWc5F7PSkY4HY,848 +numpy/f2py/tests/test_module_doc.py,sha256=sjCXWIKrqMD1NQ1DUAzgQqkjS5w9h9gvM_Lj29Rdcrg,863 +numpy/f2py/tests/test_parameter.py,sha256=ADI7EV_CM4ztICpqHqeq8LI-WdB6cX0ttatdRdjbsUA,3941 +numpy/f2py/tests/test_pyf_src.py,sha256=eD0bZu_GWfoCq--wWqEKRf-F2h5AwoTyO6GMA9wJPr4,1135 +numpy/f2py/tests/test_quoted_character.py,sha256=cpjMdrHwimnkoJkXd_W_FSlh43oWytY5VHySW9oskO4,454 +numpy/f2py/tests/test_regression.py,sha256=v_6RDQr6IcMmbCMElfzRSLPgZhHnH5l99uztrbJAzqE,2532 +numpy/f2py/tests/test_return_character.py,sha256=18HJtiRwQ7a_2mdPUonD5forKWZJEapD-Vi1DsbTjVs,1493 +numpy/f2py/tests/test_return_complex.py,sha256=BZIIqQ1abdiPLgVmu03_q37yCtND0ijxGSMhGz2Wf-o,2397 +numpy/f2py/tests/test_return_integer.py,sha256=t--9UsdLF9flLTQv7a0KTSVoBuoDtTnmOG2QIFPINVc,1758 +numpy/f2py/tests/test_return_logical.py,sha256=XCmp8E8I6BOeNYF59HjSFAdv1hM9WaDvl8UDS10_05o,2017 +numpy/f2py/tests/test_return_real.py,sha256=ATek5AM7dCCPeIvoMOQIt5yFNFzKrFb1Kno8B4M0rn4,3235 +numpy/f2py/tests/test_semicolon_split.py,sha256=_Mdsi84lES18pPjl9J-QsbGttV4tPFFjZvJvejNcqPc,1635 +numpy/f2py/tests/test_size.py,sha256=q6YqQvcyqdXJeWbGijTiCbxyEG3EkPcvT8AlAW6RCMo,1164 +numpy/f2py/tests/test_string.py,sha256=5xZOfdReoHnId0950XfmtfduPPfBbtMkzBoXMtygvMk,2962 +numpy/f2py/tests/test_symbolic.py,sha256=28quk2kTKfWhKe56n4vINJ8G9weKBfc7HysMlE9J3_g,18341 +numpy/f2py/tests/test_value_attrspec.py,sha256=rWwJBfE2qGzqilZZurJ-7ucNoJDICye6lLetQSLFees,323 +numpy/f2py/tests/util.py,sha256=bEhG699c4bLVPR2WR8fV67avgX6kH5I74SicGb7Z7T4,11167 +numpy/f2py/use_rules.py,sha256=3pTDOPur6gbPHPtwuMJPQvpnUMw39Law1KFSH0coB_0,3527 +numpy/fft/__init__.py,sha256=HqjmF6s_dh0Ri4UZzUDtOKbNUyfAfJAWew3e3EL_KUk,8175 +numpy/fft/__init__.pyi,sha256=vD9Xzz5r13caF4AVL87Y4U9KOj9ic25Vci_wb3dmgpk,550 +numpy/fft/_pocketfft.py,sha256=Xkm8wcP4JyBNMbp0ZoHIWhNDlgliX24RzrDuo29uRks,52897 +numpy/fft/_pocketfft.pyi,sha256=S6-ylUuHbgm8vNbh7tLru6K2R5SJzE81BC_Sllm6QrQ,2371 +numpy/fft/_pocketfft_internal.cpython-312-x86_64-linux-gnu.so,sha256=ONIiSfNRsdUOkmnFloif_GOGOevBUMHnX8n1Wg8zGrU,97008 +numpy/fft/helper.py,sha256=aNj1AcLvtfoX26RiLOwcR-k2QSMuBZkGj2Fu0CeFPJs,6154 +numpy/fft/helper.pyi,sha256=NLTEjy2Gz1aAMDZwCgssIyUne0ubjJqukfYkpsL3gXM,1176 +numpy/fft/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/fft/tests/test_helper.py,sha256=whgeaQ8PzFf3B1wkbXobGZ5sF4WxPp4gf1UPUVZest8,6148 +numpy/fft/tests/test_pocketfft.py,sha256=RdeCCvUQmJYVvccOJwToobTKDg9yzUL06o9MkPmRfmI,12895 +numpy/lib/__init__.py,sha256=XMPNJkG_mQ__xuvbf0OcpotgMbA9owt10ZHYVnYHq8E,2713 +numpy/lib/__init__.pyi,sha256=y5ANokFm7EkrlNoHdeQm1FsUhLFxkYtLuanCbsWrGio,5596 +numpy/lib/_datasource.py,sha256=CDF3im6IxdY3Mu6fwRQmkSEBmXS3kQVInQ4plXsoX9c,22631 +numpy/lib/_iotools.py,sha256=Yg9HCfPg4tbhbdgLPcxSMiZXq1xDprvJKLebLwhDszY,30868 +numpy/lib/_version.py,sha256=6vK7czNSB_KrWx2rZJzJ1pyOc73Q07hAgfLB5ItUCnU,4855 +numpy/lib/_version.pyi,sha256=B572hyWrUWG-TAAAXrNNAT4AgyUAmJ4lvgpwMkDzunk,633 +numpy/lib/arraypad.py,sha256=bKP7ZS9NYFYzqSk8OnpFLFrMsua4m_hcqFsi7cGkrJE,31803 +numpy/lib/arraypad.pyi,sha256=ADXphtAORYl3EqvE5qs_u32B_TALKSOtF43jOLmoxRw,1728 +numpy/lib/arraysetops.py,sha256=GJ2RhkzIJmIbwyG6h3LOFTPXg62kM9tcV1a-7tdbVuU,33655 +numpy/lib/arraysetops.pyi,sha256=6X-5l5Yss_9y10LYyIsDLbGX77vt7PtVLDqxOlSRPfY,8372 +numpy/lib/arrayterator.py,sha256=BQ97S00zvfURUZfes0GZo-5hydYNRuvwX1I1bLzeRik,7063 +numpy/lib/arrayterator.pyi,sha256=f7Pwp83_6DiMYmJGUsffncM-FRAynB1iYGvhmHM_SZE,1537 +numpy/lib/format.py,sha256=T8qJMyG2DDVjjYNNpUvBgfA9tCo23IS0w9byRB6twwQ,34769 +numpy/lib/format.pyi,sha256=YWBxC3GdsZ7SKBN8I7nMwWeVuFD1aT9d-VJ8zE4-P-o,748 +numpy/lib/function_base.py,sha256=IhhgfSmYJE-dHoUOMXHPiGYXso-NdXPpLXF9y0gEA6I,189172 +numpy/lib/function_base.pyi,sha256=KWaC5UOBANU4hiIoN2eptE4HYsm4vgp_8BMFV1Y3JX4,16585 +numpy/lib/histograms.py,sha256=xsj_qpaZoI2Bv1FBpY8mIMPJrYRiuIBszn_6kO7YFRA,37778 +numpy/lib/histograms.pyi,sha256=hNwR2xYWkgJCP-nfRGxc-EgHLTD3qm4zmWXthZLt08M,995 +numpy/lib/index_tricks.py,sha256=4PEvXk6VFTkttMViYBVC4yDhyOiKIon6JpIm0d_CmNg,31346 +numpy/lib/index_tricks.pyi,sha256=D2nkNXOB9Vea1PfMaTn94OGBGayjTaQ-bKMsjDmYpak,4251 +numpy/lib/mixins.py,sha256=y6_MzQuiNjv-1EFVROqv2y2cAJi5X4rQYzbZCyUyXgw,7071 +numpy/lib/mixins.pyi,sha256=h9N1kbZsUntF0zjOxPYeD_rCB2dMiG35TYYPl9ymkI4,3117 +numpy/lib/nanfunctions.py,sha256=6EjzydZlugIzfiENKtC4ycZ2Nckt8ZQg5v6D6tX1SiU,65775 +numpy/lib/nanfunctions.pyi,sha256=oPqAfCinmBL85Ji7ko4QlzAzLAK9nZL0t2_CllEbCEU,606 +numpy/lib/npyio.py,sha256=NUjtFvAmPdTjwJQ-ia-xbCr849M_M6NilP5IHfkKaRg,97316 +numpy/lib/npyio.pyi,sha256=SUFWJh90vWZCdd6GCSGbfYeXKlWut0XY_SHvZJc8yqY,9728 +numpy/lib/polynomial.py,sha256=6Aw3_2vdbh4urERQ6NaPhf9a_T1o1o6cjm3fb5Z3_YE,44133 +numpy/lib/polynomial.pyi,sha256=GerIpQnf5LdtFMOy9AxhOTqUyfn57k4MxqEYrfdckWE,6958 +numpy/lib/recfunctions.py,sha256=-90AbWWvVFOqVUPLh9K9NYdKUHYIgSEyg2Y35MnOVUA,59423 +numpy/lib/scimath.py,sha256=T4ITysZgqhY1J8IxyXCtioHjMTg2ci-4i3mr9TBF2UA,15037 +numpy/lib/scimath.pyi,sha256=E2roKJzMFwWSyhLu8UPUr54WOpxF8jp_pyXYBgsUSQ8,2883 +numpy/lib/setup.py,sha256=0K5NJKuvKvNEWp-EX7j0ODi3ZQQgIMHobzSFJq3G7yM,405 +numpy/lib/shape_base.py,sha256=AhCO9DEyysE-P-QJF9ryUtJ1ghU4_0mORhAJ59poObU,38947 +numpy/lib/shape_base.pyi,sha256=bGJhLA_RvUpVTiDFgCV-1rUjV8e1qCh0gK_3PLgXA_U,5341 +numpy/lib/stride_tricks.py,sha256=brY5b-0YQJuIH2CavfpIinMolyTUv5k9DUvLoZ-imis,17911 +numpy/lib/stride_tricks.pyi,sha256=0pQ4DP9l6g21q2Ajv6dJFRWMr9auPGTNV9BmZUbogPY,1747 +numpy/lib/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/lib/tests/data/py2-objarr.npy,sha256=F4cyUC-_TB9QSFLAo2c7c44rC6NUYIgrfGx9PqWPSKk,258 +numpy/lib/tests/data/py2-objarr.npz,sha256=xo13HBT0FbFZ2qvZz0LWGDb3SuQASSaXh7rKfVcJjx4,366 +numpy/lib/tests/data/py3-objarr.npy,sha256=pTTVh8ezp-lwAK3fkgvdKU8Arp5NMKznVD-M6Ex_uA0,341 +numpy/lib/tests/data/py3-objarr.npz,sha256=qQR0gS57e9ta16d_vCQjaaKM74gPdlwCPkp55P-qrdw,449 +numpy/lib/tests/data/python3.npy,sha256=X0ad3hAaLGXig9LtSHAo-BgOvLlFfPYMnZuVIxRmj-0,96 +numpy/lib/tests/data/win64python2.npy,sha256=agOcgHVYFJrV-nrRJDbGnUnF4ZTPYXuSeF-Mtg7GMpc,96 +numpy/lib/tests/test__datasource.py,sha256=65KXfUUvp8wXSqgQisuYlkhg-qHjBV5FXYetL8Ba-rc,10571 +numpy/lib/tests/test__iotools.py,sha256=HerCqvDE07JxjFQlWEfpZO7lC9z0Sbr3z20GSutoCPs,13743 +numpy/lib/tests/test__version.py,sha256=aO3YgkAohLsLzCNQ7vjIwdpFUMz0cPLbcuuxIkjuN74,1999 +numpy/lib/tests/test_arraypad.py,sha256=obohHbyM0gPYPUkd7iJSOSiDqyqtJsjDNtQX68NC4lM,54830 +numpy/lib/tests/test_arraysetops.py,sha256=5-T1MVhfIMivat8Z47GZw0ZaR811W_FskM1bAXnFyLU,35912 +numpy/lib/tests/test_arrayterator.py,sha256=AYs2SwV5ankgwnvKI9RSO1jZck118nu3SyZ4ngzZNso,1291 +numpy/lib/tests/test_financial_expired.py,sha256=yq5mqGMvqpkiiw9CuZhJgrYa7Squj1mXr_G-IvAFgwI,247 +numpy/lib/tests/test_format.py,sha256=xV0oi1eoRnVwAAhSOcPFQHQWF7TfsROtDYShQLPtdaA,41028 +numpy/lib/tests/test_function_base.py,sha256=DBKugIUEFTMP7g6iL1bk986E6ldCrcNdBCWOJbQla_Y,157830 +numpy/lib/tests/test_histograms.py,sha256=16_XJp-eFgsuM8B4mDQpQ4w_Ib29Hg0EPO-WFsdaFWA,32815 +numpy/lib/tests/test_index_tricks.py,sha256=Vjz25Y6H_ih0iEE2AG0kaxO9U8PwcXSrofzqnN4XBwI,20256 +numpy/lib/tests/test_io.py,sha256=3Tow1pucrQ7z7osNN4a2grBYUoBGNkQEhjmCjXT6Vag,107891 +numpy/lib/tests/test_loadtxt.py,sha256=gwcDJDJmLJRMLpg322yjQ1IzI505w9EqJoq4DmDPCdI,38560 +numpy/lib/tests/test_mixins.py,sha256=Wivwz3XBWsEozGzrzsyyvL3qAuE14t1BHk2LPm9Z9Zc,7030 +numpy/lib/tests/test_nanfunctions.py,sha256=01r_mmTCvKVdZuOGTEHNDZXrMS724us_jwZANzCd74A,47609 +numpy/lib/tests/test_packbits.py,sha256=OWGAd5g5GG0gl7WHqNfwkZ7G-2rrtLt2sI854PG4nnw,17546 +numpy/lib/tests/test_polynomial.py,sha256=URouxJpr8FQ5hiKybqhtOcLA7e-3hj4kWzjLBROByyA,11395 +numpy/lib/tests/test_recfunctions.py,sha256=6jzouPEQ7Uhtj8_-W5yTI6ymNp2nLgmdHzxdd74jVuM,44001 +numpy/lib/tests/test_regression.py,sha256=KzGFkhTcvEG97mymoOQ2hP2CEr2nPZou0Ztf4-WaXCs,8257 +numpy/lib/tests/test_shape_base.py,sha256=2iQCEFR6evVpF8woaenxUOzooHkfuMYkBaUj8ecyJ-E,26817 +numpy/lib/tests/test_stride_tricks.py,sha256=wprpWWH5eq07DY7rzG0WDv5fMtLxzRQz6fm6TZWlScQ,22849 +numpy/lib/tests/test_twodim_base.py,sha256=ll-72RhqCItIPB97nOWhH7H292h4nVIX_w1toKTPMUg,18841 +numpy/lib/tests/test_type_check.py,sha256=lxCH5aApWVYhhSoDQSLDTCHLVHuK2c-jBbnfnZUrOaA,15114 +numpy/lib/tests/test_ufunclike.py,sha256=4hSnXGlSC8HE-_pRRMzD8-HI4hGHqsAWu1pD0o2kPI0,2982 +numpy/lib/tests/test_utils.py,sha256=RVAxrzSFu6N3C4_jIgAlTDOWF_B7wr2v1Y20dX5upYM,6218 +numpy/lib/twodim_base.py,sha256=Mvzn_PyShIb9m7nJjJ4IetdxwmLYEsCPHvJoK7n2viU,32947 +numpy/lib/twodim_base.pyi,sha256=xFRcEVJdDj4mrXW_6iVP1lTMoJx4QJjYRD3o2_9f2eY,5370 +numpy/lib/type_check.py,sha256=_EOtB296nFYlNT7ztBYoC_yK9aycIb0KTmRjvzVdZNg,19954 +numpy/lib/type_check.pyi,sha256=LPvAvIxU-p5i_Qe-ic7hEvo4OTfSrNpplxMG7OAZe8Q,5571 +numpy/lib/ufunclike.py,sha256=_ceBGbGCMOd3u_h2UVzyaRK6ZY7ryoJ0GJB7zqcJG3w,6325 +numpy/lib/ufunclike.pyi,sha256=hLxcYfQprh1tTY_UO2QscA3Hd9Zd7cVGXIINZLhMFqY,1293 +numpy/lib/user_array.py,sha256=LE958--CMkBI2r3l1SQxmCHdCSw6HY6-RhWCnduzGA4,7721 +numpy/lib/utils.py,sha256=6NdleaELZiqARdj-ECZjxtwLf1bqklOcK43m9yoZefs,37804 +numpy/lib/utils.pyi,sha256=mVHVzWuc2-M3Oz60lFsbok0v8LH_HRHMjZpXwrtzF_c,2360 +numpy/linalg/__init__.py,sha256=mpdlEXWtTvpF7In776ONLwp6RIyo4U_GLPT1L1eIJnw,1813 +numpy/linalg/__init__.pyi,sha256=XBy4ocuypsRVflw_mbSTUhR4N5Roemu6w5SfeVwbkAc,620 +numpy/linalg/_umath_linalg.cpython-312-x86_64-linux-gnu.so,sha256=iCLnctdD1AWYPxucazS3BN0pd4CJDcJFRU8Qga31Ckw,216793 +numpy/linalg/lapack_lite.cpython-312-x86_64-linux-gnu.so,sha256=UAZPuN2wY1u7YCi4990o-QwErZqxw_rd0RF8K7fcj_0,29849 +numpy/linalg/linalg.py,sha256=kDVK1GBxbUjlRgxXCoEfkRJm8yrNr1Iu7hMn2rKK8RE,90923 +numpy/linalg/linalg.pyi,sha256=zD9U5BUCB1uQggSxfZaTGX_uB2Hkp75sttGmZbCGgBI,7505 +numpy/linalg/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/linalg/tests/test_deprecations.py,sha256=9p_SRmtxj2zc1doY9Ie3dyy5JzWy-tCQWFoajcAJUmM,640 +numpy/linalg/tests/test_linalg.py,sha256=rgvmK6Or70u8mN04puetL3FgSxZ8fJrOlI5ptTgCU5k,78085 +numpy/linalg/tests/test_regression.py,sha256=qbugUmrENybkEaM1GhfA01RXQUy8AkzalbrfzSIgUmM,5434 +numpy/ma/API_CHANGES.txt,sha256=F_4jW8X5cYBbzpcwteymkonTmvzgKKY2kGrHF1AtnrI,3405 +numpy/ma/LICENSE,sha256=BfO4g1GYjs-tEKvpLAxQ5YdcZFLVAJoAhMwpFVH_zKY,1593 +numpy/ma/README.rst,sha256=q-gCsZ4Cw_gUGGvEjog556sJUHIm8WTAwkFK5Qnz9XA,9872 +numpy/ma/__init__.py,sha256=dgP0WdnOpph28Fd6UiqoyDKhfrct0H6QWqbCcETsk6M,1404 +numpy/ma/__init__.pyi,sha256=ppCg_TS0POutNB3moJE4kBabWURnc0WGXyYPquXZxS4,6063 +numpy/ma/core.py,sha256=4MglVRJtmQ9_iIVaQ2b-_Vmw1TjAhEsMJdtKOhyBFXQ,278213 +numpy/ma/core.pyi,sha256=YfgyuBuKxZ5v4I2JxZDvCLhnztOCRgzTeDg-JGTon_M,14305 +numpy/ma/extras.py,sha256=MC7QPS34PC4wxNbOp7pTy57dqF9B-L6L1KMI6rrfe2w,64383 +numpy/ma/extras.pyi,sha256=BBsiCZbaPpGCY506fkmqZdBkJNCXcglc3wcSBuAACNk,2646 +numpy/ma/mrecords.py,sha256=degd6dLaDEvEWNHmvSnUZXos1csIzaqjR_jAutm8JfI,27232 +numpy/ma/mrecords.pyi,sha256=r1a2I662ywnhGS6zvfcyK-9RHVvb4sHxiCx9Dhf5AE4,1934 +numpy/ma/setup.py,sha256=MqmMicr_xHkAGoG-T7NJ4YdUZIJLO4ZFp6AmEJDlyhw,418 +numpy/ma/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/ma/tests/test_core.py,sha256=xd5S3oa0jObo8jnsJk0-o46d-KNC3RtgNRKinJeY_kE,215100 +numpy/ma/tests/test_deprecations.py,sha256=nq_wFVt2EBHcT3AHxattfKXx2JDf1K5D-QBzUU0_15A,2566 +numpy/ma/tests/test_extras.py,sha256=lX4cbdGDEXaBHzA3q8hJxve4635XCJw4AP7FO7zhOfk,74858 +numpy/ma/tests/test_mrecords.py,sha256=PsJhUlABgdpSsPUeijonfyFNqz5AfNSGQTtJUte7yts,19890 +numpy/ma/tests/test_old_ma.py,sha256=h4BncexBcBigqvZMA6RjDjpHPurWtt99A7KTag2rmOs,32690 +numpy/ma/tests/test_regression.py,sha256=foMpI0luAvwkkRpAfPDV_810h1URISXDZhmaNhxb50k,3287 +numpy/ma/tests/test_subclassing.py,sha256=HeTIE_n1I8atwzF8tpvNtGHp-0dmM8PT8AS4IDWbcso,16967 +numpy/ma/testutils.py,sha256=RQw0RyS7hOSVTk4KrCGleq0VHlnDqzwwaLtuZbRE4_I,10235 +numpy/ma/timer_comparison.py,sha256=pIGSZG-qYYYlRWSTgzPlyCAINbGKhXrZrDZBBjiM080,15658 +numpy/matlib.py,sha256=-54vTuGIgeTMg9ZUmElRPZ4Hr-XZ-om9xLzAsSoTvnc,10465 +numpy/matrixlib/__init__.py,sha256=BHBpQKoQv4EjT0UpWBA-Ck4L5OsMqTI2IuY24p-ucXk,242 +numpy/matrixlib/__init__.pyi,sha256=-t3ZuvbzRuRwWfZOeN4xlNWdm7gQEprhUsWzu8MRvUE,252 +numpy/matrixlib/defmatrix.py,sha256=JXdJGm1LayOOXfKpp7OVZfb0pzzP4Lwh45sTJrleALc,30656 +numpy/matrixlib/defmatrix.pyi,sha256=lmBMRahKcMOl2PHDo79J67VRAZOkI54BzfDaTLpE0LI,451 +numpy/matrixlib/setup.py,sha256=1r7JRkSM4HyVorgtjoKJGWLcOcPO3wmvivpeEsVtAEg,426 +numpy/matrixlib/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/matrixlib/tests/test_defmatrix.py,sha256=8E_-y7VD2vsq1y8CcI8km37pp5qcAtkciO16xqf2UIs,14982 +numpy/matrixlib/tests/test_interaction.py,sha256=PpjmgjEKighDXvt38labKE6L7f2jP74UEmp3JRb_iOY,11875 +numpy/matrixlib/tests/test_masked_matrix.py,sha256=7YO_LCO8DOhW3CuXJuxH93rnmttfvHnU7El-MBzxzFw,8932 +numpy/matrixlib/tests/test_matrix_linalg.py,sha256=ObbSUXU4R2pWajH__xAdizADrU2kBKDDCxkDV-oVBXc,2059 +numpy/matrixlib/tests/test_multiarray.py,sha256=jB3XCBmAtcqf-Wb9PwBW6uIykPpMPthuXLJ0giTKzZE,554 +numpy/matrixlib/tests/test_numeric.py,sha256=MP70qUwgshTtThKZaZDp7_6U-Z66NIV1geVhasGXejQ,441 +numpy/matrixlib/tests/test_regression.py,sha256=8sHDtO8Zi8p3a1eQKEWxtCmKrXmHoD3qxlIokg2AIAU,927 +numpy/polynomial/__init__.py,sha256=braLh6zP2QwuNKRKAaZGdC_qKWZ-tJlc3BN83LeuE_0,6781 +numpy/polynomial/__init__.pyi,sha256=W8szYtVUy0RUi83jmFLK58BN8CKVSoHA2CW7IcdUl1c,701 +numpy/polynomial/_polybase.py,sha256=YEnnQwlTgbn3dyD89ueraUx5nxx3x_pH6K6mmyEmhi8,39271 +numpy/polynomial/_polybase.pyi,sha256=J7yU9PPZW4W8mkqAltDfnL4ZNwljuM-bDEj4DPTJZpY,2321 +numpy/polynomial/chebyshev.py,sha256=NZCKjIblcX99foqZyp51i0_r8p0r1VKVGZFmQ1__kEk,62796 +numpy/polynomial/chebyshev.pyi,sha256=035CNdOas4dnb6lFLzRiBrYT_VnWh2T1-A3ibm_HYkI,1387 +numpy/polynomial/hermite.py,sha256=t5CFM-qE4tszYJiQZ301VcMn7IM67y2rUZPFPtnVRAc,52514 +numpy/polynomial/hermite.pyi,sha256=hdsvTULow8bIjnATudf0i6brpLHV7vbOoHzaMvbjMy0,1217 +numpy/polynomial/hermite_e.py,sha256=jRR3f8Oth8poV2Ix8c0eLEQR3UZary-2RupOrEAEUMY,52642 +numpy/polynomial/hermite_e.pyi,sha256=zV7msb9v9rV0iv_rnD3SjP-TGyc6pd3maCqiPCj3PbA,1238 +numpy/polynomial/laguerre.py,sha256=mcVw0ckWVX-kzJ1QIhdcuuxzPjuFmA3plQLkloQMOYM,50858 +numpy/polynomial/laguerre.pyi,sha256=Gxc9SLISNKMWrKdsVJ9fKFFFwfxxZzfF-Yc-2r__z5M,1178 +numpy/polynomial/legendre.py,sha256=wjtgFajmKEbYkSUk3vWSCveMHDP6UymK28bNUk4Ov0s,51550 +numpy/polynomial/legendre.pyi,sha256=9dmANwkxf7EbOHV3XQBPoaDtc56cCkf75Wo7FG9Zfj4,1178 +numpy/polynomial/polynomial.py,sha256=XsaZPHmLGJFqpJs7rPvO5E0loWQ1L3YHLIUybVu4dU8,49112 +numpy/polynomial/polynomial.pyi,sha256=bOPRnub4xXxsUwNGeiQLTT4PCfN1ysSrf6LBZIcAN2Y,1132 +numpy/polynomial/polyutils.py,sha256=Xy5qjdrjnRaqSlClG1ROmwWccLkAPC7IcHaNJLvhCf4,23237 +numpy/polynomial/polyutils.pyi,sha256=cFAyZ9Xzuw8Huhn9FEz4bhyD00m2Dp-2DiUSyogJwSo,264 +numpy/polynomial/setup.py,sha256=dXQfzVUMP9OcB6iKv5yo1GLEwFB3gJ48phIgo4N-eM0,373 +numpy/polynomial/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/polynomial/tests/test_chebyshev.py,sha256=6tMsFP1h7K8Zf72mNOta6Tv52_fVTlXknseuffj080c,20522 +numpy/polynomial/tests/test_classes.py,sha256=DFyY2IQBj3r2GZkvbRIeZO2EEY466xbuwc4PShAl4Sw,18331 +numpy/polynomial/tests/test_hermite.py,sha256=N9b2dx2UWPyja5v02dSoWYPnKvb6H-Ozgtrx-xjWz2k,18577 +numpy/polynomial/tests/test_hermite_e.py,sha256=_A3ohAWS4HXrQG06S8L47dImdZGTwYosCXnoyw7L45o,18911 +numpy/polynomial/tests/test_laguerre.py,sha256=BZOgs49VBXOFBepHopxuEDkIROHEvFBfWe4X73UZhn8,17511 +numpy/polynomial/tests/test_legendre.py,sha256=b_bblHs0F_BWw9ESuSq52ZsLKcQKFR5eqPf_SppWFqo,18673 +numpy/polynomial/tests/test_polynomial.py,sha256=4cuO8-5wdIxcz5CrucB5Ix7ySuMROokUF12F7ogQ_hc,20529 +numpy/polynomial/tests/test_polyutils.py,sha256=IxkbVfpcBqe5lOZluHFUPbLATLu1rwVg7ghLASpfYrY,3579 +numpy/polynomial/tests/test_printing.py,sha256=rfP4MaQbjGcO52faHmYrgsaarkm3Ndi3onwr6DDuapE,20525 +numpy/polynomial/tests/test_symbol.py,sha256=msTPv7B1niaKujU33kuZmdxJvLYvOjfl1oykmlL0dXo,5371 +numpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/random/LICENSE.md,sha256=EDFmtiuARDr7nrNIjgUuoGvgz_VmuQjxmeVh_eSa8Z8,3511 +numpy/random/__init__.pxd,sha256=9JbnX540aJNSothGs-7e23ozhilG6U8tINOUEp08M_k,431 +numpy/random/__init__.py,sha256=81Thnexg5umN5WZwD5TRyzNc2Yp-d14B6UC7NBgVKh8,7506 +numpy/random/__init__.pyi,sha256=RfW8mco48UaWDL1UC5ROv9vXiFZ9EGho62avhgEAHPc,2143 +numpy/random/_bounded_integers.cpython-312-x86_64-linux-gnu.so,sha256=s59-K0zP1pBK5g_hUX9r2ovng1tb9p1U3sDWk8Xot5M,348704 +numpy/random/_bounded_integers.pxd,sha256=hcoucPH5hkFEM2nm12zYO-5O_Rt8RujEXT5YWuAzl1Q,1669 +numpy/random/_common.cpython-312-x86_64-linux-gnu.so,sha256=q9iMqPRH8ixPUfImc000cylmuuYe3SqiX3S_7JVL7ig,258888 +numpy/random/_common.pxd,sha256=s2_IdIQ0MhNbogamulvXe-b93wbx882onmYkxqswwpo,4939 +numpy/random/_examples/cffi/extending.py,sha256=xSla3zWqxi6Hj48EvnYfD3WHfE189VvC4XsKu4_T_Iw,880 +numpy/random/_examples/cffi/parse.py,sha256=Bnb7t_6S_c5-3dZrQ-XX9EazOKhftUfcCejXXWyd1EU,1771 +numpy/random/_examples/cython/extending.pyx,sha256=4IE692pq1V53UhPZqQiQGcIHXDoNyqTx62x5a36puVg,2290 +numpy/random/_examples/cython/extending_distributions.pyx,sha256=oazFVWeemfE0eDzax7r7MMHNL1_Yofws2m-c_KT2Hbo,3870 +numpy/random/_examples/cython/meson.build,sha256=rXtugURMEo-ef4bPE1QIv4mzvWbeGjmcTdKCBvjxjtw,1443 +numpy/random/_examples/numba/extending.py,sha256=Ipyzel_h5iU_DMJ_vnXUgQC38uMDMn7adUpWSeEQLFE,1957 +numpy/random/_examples/numba/extending_distributions.py,sha256=Jnr9aWkHyIWygNbdae32GVURK-5T9BTGhuExRpvve98,2034 +numpy/random/_generator.cpython-312-x86_64-linux-gnu.so,sha256=Wz7yrIt4qoO8hptw4w4qcPvTqzc8UlPtbrqZgqVf1-I,946872 +numpy/random/_generator.pyi,sha256=zRvo_y6g0pWkE4fO1M9jLYUkxDfGdA6Enreb3U2AADM,22442 +numpy/random/_mt19937.cpython-312-x86_64-linux-gnu.so,sha256=Nhn3-Rue5xl8KQLA4Zfmmy5d1F-xHNIuVy6bC4hlFKk,119488 +numpy/random/_mt19937.pyi,sha256=_iZKaAmuKBQ4itSggfQvYYj_KjktcN4rt-YpE6bqFAM,724 +numpy/random/_pcg64.cpython-312-x86_64-linux-gnu.so,sha256=V3wUaPT7QLsjGEND4sG2RaF9HUk2QeqSwFLyhtxutVY,125040 +numpy/random/_pcg64.pyi,sha256=uxr5CbEJetN6lv9vBG21jlRhuzOK8SQnXrwqAQBxj_c,1091 +numpy/random/_philox.cpython-312-x86_64-linux-gnu.so,sha256=LJsf5T7xGePtKstzyALPKZZQKw_VHUkm1AR1ds6ldRQ,106712 +numpy/random/_philox.pyi,sha256=OKlaiIU-hj72Bp04zjNifwusOD_3-mYxIfvyuys8c_o,978 +numpy/random/_pickle.py,sha256=4NhdT-yk7C0m3tyZWmouYAs3ZGNPdPVNGfUIyuh8HDY,2318 +numpy/random/_sfc64.cpython-312-x86_64-linux-gnu.so,sha256=WIMwLOM6_VTbZjGtv14AApe460LA7IlLvsMYteaxQmg,76224 +numpy/random/_sfc64.pyi,sha256=09afHTedVW-519493ZXtGcl-H-_zluj-B_yfEJG8MMs,709 +numpy/random/bit_generator.cpython-312-x86_64-linux-gnu.so,sha256=h8XHMIh5Q8YDsWxTzyEzmDNu5BDvRllVGAk5d6_VsMs,234016 +numpy/random/bit_generator.pxd,sha256=lArpIXSgTwVnJMYc4XX0NGxegXq3h_QsUDK6qeZKbNc,1007 +numpy/random/bit_generator.pyi,sha256=aXv7a_hwa0nkjY8P2YENslwWp89UcFRn09woXh7Uoc0,3510 +numpy/random/c_distributions.pxd,sha256=7DE-mV3H_Dihk4OK4gMHHkyD4tPX1cAi4570zi5CI30,6344 +numpy/random/lib/libnpyrandom.a,sha256=xUcvOvieju5PThPQ8q0-uGJ5fjsCd5umnjIerIc85Sg,71926 +numpy/random/mtrand.cpython-312-x86_64-linux-gnu.so,sha256=jYZrS2EHQBq5VGBEVkII4KWJnDkD2gknfRxIddmLzw8,749040 +numpy/random/mtrand.pyi,sha256=3vAGOXsvyFFv0yZl34pVVPP7Dgt22COyfn4tUoi_hEQ,19753 +numpy/random/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/random/tests/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/random/tests/data/mt19937-testset-1.csv,sha256=Xkef402AVB-eZgYQkVtoxERHkxffCA9Jyt_oMbtJGwY,15844 +numpy/random/tests/data/mt19937-testset-2.csv,sha256=nsBEQNnff-aFjHYK4thjvUK4xSXDSfv5aTbcE59pOkE,15825 +numpy/random/tests/data/pcg64-testset-1.csv,sha256=xB00DpknGUTTCxDr9L6aNo9Hs-sfzEMbUSS4t11TTfE,23839 +numpy/random/tests/data/pcg64-testset-2.csv,sha256=NTdzTKvG2U7_WyU_IoQUtMzU3kEvDH39CgnR6VzhTkw,23845 +numpy/random/tests/data/pcg64dxsm-testset-1.csv,sha256=vNSUT-gXS_oEw_awR3O30ziVO4seNPUv1UIZ01SfVnI,23833 +numpy/random/tests/data/pcg64dxsm-testset-2.csv,sha256=uylS8PU2AIKZ185OC04RBr_OePweGRtvn-dE4YN0yYA,23839 +numpy/random/tests/data/philox-testset-1.csv,sha256=SedRaIy5zFadmk71nKrGxCFZ6BwKz8g1A9-OZp3IkkY,23852 +numpy/random/tests/data/philox-testset-2.csv,sha256=dWECt-sbfvaSiK8-Ygp5AqyjoN5i26VEOrXqg01rk3g,23838 +numpy/random/tests/data/sfc64-testset-1.csv,sha256=iHs6iX6KR8bxGwKk-3tedAdMPz6ZW8slDSUECkAqC8Q,23840 +numpy/random/tests/data/sfc64-testset-2.csv,sha256=FIDIDFCaPZfWUSxsJMAe58hPNmMrU27kCd9FhCEYt_k,23833 +numpy/random/tests/test_direct.py,sha256=6vLpCyeKnAWFEZei7l2YihVLQ0rSewO1hJBWt7A5fyQ,17779 +numpy/random/tests/test_extending.py,sha256=S3Wrzu3di4uBhr-Pxnx5dOPvlBY0FRdZqVX6CC1IN6s,4038 +numpy/random/tests/test_generator_mt19937.py,sha256=35LBwV6TtWPnxhefutxTQmhLzAQ5Ee4YiY8ziDXM-eQ,115477 +numpy/random/tests/test_generator_mt19937_regressions.py,sha256=xGkdz76BMX1EK0QPfabVxpNx9qQ9OC-1ZStWOs6N_M8,6387 +numpy/random/tests/test_random.py,sha256=kEkQs3i7zcpm9MozIRIz1FIx5B6fmXk0QqX0l6l-u_Y,70087 +numpy/random/tests/test_randomstate.py,sha256=DxF7rMUSxaAlL4h1qC3onHcHR7T_6rKWPbr0nJH84nE,85031 +numpy/random/tests/test_randomstate_regression.py,sha256=VucYWIjA7sAquWsalvZMnfkmYLM1O6ysyWnLl931-lA,7917 +numpy/random/tests/test_regression.py,sha256=trntK51UvajOVELiluEO85l64CKSw5nvBSc5SqYyr9w,5439 +numpy/random/tests/test_seed_sequence.py,sha256=GNRJ4jyzrtfolOND3gUWamnbvK6-b_p1bBK_RIG0sfU,3311 +numpy/random/tests/test_smoke.py,sha256=jjNz0aEGD1_oQl9a9UWt6Mz_298alG7KryLT1pgHljw,28183 +numpy/testing/__init__.py,sha256=InpVKoDAzMKO_l_HNcatziW_u1k9_JZze__t2nybrL0,595 +numpy/testing/__init__.pyi,sha256=AhK5NuOpdD-JjIzXOlssE8_iSLyFAAHzyGV_w1BT7vA,1674 +numpy/testing/_private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/testing/_private/extbuild.py,sha256=nG2dwP4nUmQS3e5eIRinxt0s_f4sxxA1YfohCg-navo,8017 +numpy/testing/_private/utils.py,sha256=3FrSTMi0OdpDODBDoncgiDQzdo5NKA6YVfQ3uKRSQnc,85242 +numpy/testing/_private/utils.pyi,sha256=MMNrvwEeSTYzZFWawSSzHnTFYG-cSAIiID-1FuJ1f8U,10123 +numpy/testing/overrides.py,sha256=u6fcKSBC8HIzMPWKAbdyowU71h2Fx2ekDQxpG5NhIr8,2123 +numpy/testing/print_coercion_tables.py,sha256=ndxOsS4XfrZ4UY_9nqRTCnxhkzgdqcuUHL8nezd7Op4,6180 +numpy/testing/setup.py,sha256=GPKAtTTBRsNW4kmR7NjP6mmBR_GTdpaTvkTm10_VcLg,709 +numpy/testing/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/testing/tests/test_utils.py,sha256=IDOr-GXuNGlrsb-XzGSYUHXEqcGYJ78p60jOpBqyPM4,55740 +numpy/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/tests/test__all__.py,sha256=L3mCnYPTpzAgNfedVuq9g7xPWbc0c1Pot94k9jZ9NpI,221 +numpy/tests/test_ctypeslib.py,sha256=B06QKeFRgDIEbkEPBy_zYA1H5E2exuhTi7IDkzV8gfo,12257 +numpy/tests/test_lazyloading.py,sha256=YETrYiDLAqLX04K_u5_3NVxAfxDoeguxwkIRfz6qKcY,1162 +numpy/tests/test_matlib.py,sha256=gwhIXrJJo9DiecaGLCHLJBjhx2nVGl6yHq80AOUQSRM,1852 +numpy/tests/test_numpy_config.py,sha256=qHvepgi9oyAbQuZD06k7hpcCC2MYhdzcY6D1iQDPNMI,1241 +numpy/tests/test_numpy_version.py,sha256=A8cXFzp4k-p6J5zkOxlDfDvkoFMxDW2hpTFVXcaQRVo,1479 +numpy/tests/test_public_api.py,sha256=DTq7SO84uBjC2tKPoqX17xazc-SLkTAbQ2fLZwGM2jc,18170 +numpy/tests/test_reloading.py,sha256=QuVaPQulcNLg4Fl31Lw-O89L42KclYCK68n5GVy0PNQ,2354 +numpy/tests/test_scripts.py,sha256=jluCLfG94VM1cuX-5RcLFBli_yaJZpIvmVuMxRKRJrc,1645 +numpy/tests/test_warnings.py,sha256=ZEtXqHI1iyeVeLfVxDcMfN5qw67Ti2u54709hvBG4eY,2284 +numpy/typing/__init__.py,sha256=VoTILNDrUWvZx0LK9_97lBLQFKtSGmDt4QLOH8zYvlo,5234 +numpy/typing/mypy_plugin.py,sha256=24zVk4Ei3qH4Hc3SSz3v0XtIsycTo8HKoY6ilhB_7AQ,6376 +numpy/typing/setup.py,sha256=Cnz9q53w-vJNyE6vYxqYvQXx0pJbrG9quHyz9sqxfek,374 +numpy/typing/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +numpy/typing/tests/data/fail/arithmetic.pyi,sha256=4rY_ASCERAl8WCus1RakOe0Aw-8vvjilL29mgdD4lv0,3850 +numpy/typing/tests/data/fail/array_constructors.pyi,sha256=X9y_jUYS17WfYmXW5NwkVudyiR6ouUaAwEh0JRte42o,1089 +numpy/typing/tests/data/fail/array_like.pyi,sha256=OVAlEJZ5k8ZRKt0aGpZQwIjlUGpy0PzOOYqfI-IMqBQ,455 +numpy/typing/tests/data/fail/array_pad.pyi,sha256=57oK0Yp53rtKjjIrRFYLcxa-IfIGhtI-bEem7ggJKwI,132 +numpy/typing/tests/data/fail/arrayprint.pyi,sha256=-Fs9VnQfxyfak008Hq8kJWfB0snA6jGDXZz8ljQnwGE,549 +numpy/typing/tests/data/fail/arrayterator.pyi,sha256=FoU4ahHkJZ67dwWXer5FXLjjjesKKg-w2Jq1X1bHymA,480 +numpy/typing/tests/data/fail/bitwise_ops.pyi,sha256=GN9dVqk4_HFXn7zbRrHzJq_UGRFBccoYVUG1UuE7bXs,515 +numpy/typing/tests/data/fail/char.pyi,sha256=-vgN6EmfQ8VaA4SOZ5Ol9u4-Z7Q5I7G78LmaxZOuZ90,2615 +numpy/typing/tests/data/fail/chararray.pyi,sha256=jrNryZFpr8nxG2IHb9e0x3ranpvJpBy_RDex-WpT5rU,2296 +numpy/typing/tests/data/fail/comparisons.pyi,sha256=U4neWzwwtxG6QXsKlNGJuKXHBtwzYBQOa47_7SKF5Wg,888 +numpy/typing/tests/data/fail/constants.pyi,sha256=YSqNbXdhbdMmYbs7ntH0FCKbnm8IFeqsDlZBqcU43iw,286 +numpy/typing/tests/data/fail/datasource.pyi,sha256=PRT2hixR-mVxr2UILvHa99Dr54EF2h3snJXE-v3rWcc,395 +numpy/typing/tests/data/fail/dtype.pyi,sha256=OAGABqdXNB8gClJFEGMckoycuZcIasMaAlS2RkiKROI,334 +numpy/typing/tests/data/fail/einsumfunc.pyi,sha256=RS7GZqUCT_vEFJoyUx4gZlPO8GNFFNFWidxl-wLyRv0,539 +numpy/typing/tests/data/fail/false_positives.pyi,sha256=Q61qMsSsNCtmO0EMRxHj5Z7RYTyrELVpkzfJY5eK8Z0,366 +numpy/typing/tests/data/fail/flatiter.pyi,sha256=qLM4qm7gvJtEZ0rTHcyasUzoP5JbX4FREtqV3g1w6Lo,843 +numpy/typing/tests/data/fail/fromnumeric.pyi,sha256=FH2mjkgtCbA9soqlJRhYN7IIfRRrUL1i9mwqcbYKZSc,5591 +numpy/typing/tests/data/fail/histograms.pyi,sha256=yAPVt0rYTwtxnigoGT-u7hhKCE9iYxsXc24x2HGBrmA,367 +numpy/typing/tests/data/fail/index_tricks.pyi,sha256=moINir9iQoi6Q1ZuVg5BuSB9hSBtbg_uzv-Qm_lLYZk,509 +numpy/typing/tests/data/fail/lib_function_base.pyi,sha256=6y9T773CBLX-jUry1sCQGVuKVKM2wMuQ56Ni5V5j4Dw,2081 +numpy/typing/tests/data/fail/lib_polynomial.pyi,sha256=Ur7Y4iZX6WmoH5SDm0ePi8C8LPsuPs2Yr7g7P5O613g,899 +numpy/typing/tests/data/fail/lib_utils.pyi,sha256=VFpE6_DisvlDByyp1PiNPJEe5IcZp8cH0FlAJyoZipo,276 +numpy/typing/tests/data/fail/lib_version.pyi,sha256=7-ZJDZwDcB-wzpMN8TeYtZAgaqc7xnQ8Dnx2ISiX2Ts,158 +numpy/typing/tests/data/fail/linalg.pyi,sha256=yDd05aK1dI37RPt3pD2eJYo4dZFaT2yB1PEu3K0y9Tg,1322 +numpy/typing/tests/data/fail/memmap.pyi,sha256=HSTCQYNuW1Y6X1Woj361pN4rusSPs4oDCXywqk20yUo,159 +numpy/typing/tests/data/fail/modules.pyi,sha256=_ek4zKcdP-sIh_f-IDY0tP-RbLORKCSWelM9AOYxsyA,670 +numpy/typing/tests/data/fail/multiarray.pyi,sha256=XCdBxufNhR8ZtG8UMzk8nt9_NC5gJTKP9-xTqKO_K9I,1693 +numpy/typing/tests/data/fail/ndarray.pyi,sha256=YnjXy16RHs_esKelMjB07865CQ7gLyQnXhnitq5Kv5c,405 +numpy/typing/tests/data/fail/ndarray_misc.pyi,sha256=w-10xTDDWoff9Lq0dBO-jBeiBR-XjCz2qmes0dLx238,1372 +numpy/typing/tests/data/fail/nditer.pyi,sha256=w7emjnOxnf3NcvLktNLlke6Cuivn2gU3sVmGCfbG6rw,325 +numpy/typing/tests/data/fail/nested_sequence.pyi,sha256=em4GZwLDFE0QSxxg081wVwhh-Dmtkn8f7wThI0DiXVs,427 +numpy/typing/tests/data/fail/npyio.pyi,sha256=56QuHo9SvVR3Uhzl6gQZncCpX575Gy5wugjMICh20m0,620 +numpy/typing/tests/data/fail/numerictypes.pyi,sha256=fevH9x80CafYkiyBJ7LMLVl6GyTvQrZ34trBu6O8TtM,276 +numpy/typing/tests/data/fail/random.pyi,sha256=p5WsUGyOL-MGIeALh9Y0dVhYSRQLaUwMdjXc3G6C_7Q,2830 +numpy/typing/tests/data/fail/rec.pyi,sha256=Ws3TyesnoQjt7Q0wwtpShRDJmZCs2jjP17buFMomVGA,704 +numpy/typing/tests/data/fail/scalars.pyi,sha256=o91BwSfzPTczYVtbXsirqQUoUoYP1C_msGjc2GYsV04,2952 +numpy/typing/tests/data/fail/shape_base.pyi,sha256=Y_f4buHtX2Q2ZA4kaDTyR8LErlPXTzCB_-jBoScGh_Q,152 +numpy/typing/tests/data/fail/stride_tricks.pyi,sha256=IjA0Xrnx0lG3m07d1Hjbhtyo1Te5cXgjgr5fLUo4LYQ,315 +numpy/typing/tests/data/fail/testing.pyi,sha256=e7b5GKTWCtKGoB8z2a8edsW0Xjl1rMheALsvzEJjlCw,1370 +numpy/typing/tests/data/fail/twodim_base.pyi,sha256=ZqbRJfy5S_pW3fFLuomy4L5SBNqj6Nklexg9KDTo65c,899 +numpy/typing/tests/data/fail/type_check.pyi,sha256=CIyI0j0Buxv0QgCvNG2urjaKpoIZ-ZNawC2m6NzGlbo,379 +numpy/typing/tests/data/fail/ufunc_config.pyi,sha256=ukA0xwfJHLoGfoOIpWIN-91wj-DG8oaIjYbO72ymjg4,733 +numpy/typing/tests/data/fail/ufunclike.pyi,sha256=lbxjJyfARmt_QK1HxhxFxvwQTqCEZwJ9I53Wp8X3KIY,679 +numpy/typing/tests/data/fail/ufuncs.pyi,sha256=YaDTL7QLmGSUxE6JVMzpOlZTjHWrgbOo0UIlkX-6ZQk,1347 +numpy/typing/tests/data/fail/warnings_and_errors.pyi,sha256=PrbYDFI7IGN3Gf0OPBkVfefzQs4AXHwDQ495pvrX3RY,174 +numpy/typing/tests/data/misc/extended_precision.pyi,sha256=bS8bBeCFqjgtOiy-8_y39wfa7rwhdjLz2Vmo-RXAYD4,884 +numpy/typing/tests/data/mypy.ini,sha256=Ynv1VSx_kXTD2mFC3ZpgEFuCOg1F2VJXxPk0dxUnF2M,108 +numpy/typing/tests/data/pass/arithmetic.py,sha256=2z3dmuysQQmiPz8x0bg8SOOKW62mVJn97uMa9T0L7Vk,7455 +numpy/typing/tests/data/pass/array_constructors.py,sha256=3GrhfBcmWX53pJHD0NvhXjwr2-uNKREbR1I9WCcZ7rI,2419 +numpy/typing/tests/data/pass/array_like.py,sha256=ce_IVubBd7J6FkSpJmD7qMlRLuwmiidhOqhYfZb16Wo,916 +numpy/typing/tests/data/pass/arrayprint.py,sha256=y_KkuLz1uM7pv53qfq7GQOuud4LoXE3apK1wtARdVyM,766 +numpy/typing/tests/data/pass/arrayterator.py,sha256=FqcpKdUQBQ0FazHFxr9MsLEZG-jnJVGKWZX2owRr4DQ,393 +numpy/typing/tests/data/pass/bitwise_ops.py,sha256=UnmxVr9HwI8ifdrutGm_u3EZU4iOOPQhrOku7hTaH0c,970 +numpy/typing/tests/data/pass/comparisons.py,sha256=nTE-fvraLK6xTZcP4uPV02wOShzYKWDaoapx35AeDOY,2992 +numpy/typing/tests/data/pass/dtype.py,sha256=MqDKC6Ywv6jNkWsR8rdLuabzHUco5w1OylDHEdxve_I,1069 +numpy/typing/tests/data/pass/einsumfunc.py,sha256=eXj5L5MWPtQHgrHPsJ36qqrmBHqct9UoujjJCvHnF1k,1370 +numpy/typing/tests/data/pass/flatiter.py,sha256=0BnbuLMBC7MQlprNZ0QhNSscfYwPhEhXOhWoyiRACWU,174 +numpy/typing/tests/data/pass/fromnumeric.py,sha256=Xd_nJVVDoONdztUX8ddgo7EXJ2FD8AX51MO_Yujnmog,3742 +numpy/typing/tests/data/pass/index_tricks.py,sha256=oaFD9vY01_RI5OkrXt-xTk1n_dd-SpuPp-eZ58XR3c8,1492 +numpy/typing/tests/data/pass/lib_utils.py,sha256=sDQCjHVGUwct0RQqAtH5_16y241siSY4bXKZRsuJ8xA,434 +numpy/typing/tests/data/pass/lib_version.py,sha256=HnuGOx7tQA_bcxFIJ3dRoMAR0fockxg4lGqQ4g7LGIw,299 +numpy/typing/tests/data/pass/literal.py,sha256=DLzdWHD6ttW4S0NEvGQbsH_UEJjhZyhvO4OXJjoyvZQ,1331 +numpy/typing/tests/data/pass/mod.py,sha256=HB9aK4_wGJbc44tomaoroNy0foIL5cI9KIjknvMTbkk,1578 +numpy/typing/tests/data/pass/modules.py,sha256=t0KJxYWbrWd7HbbgIDFb3LAhJBiNNb6QPjjFDAgC2mU,576 +numpy/typing/tests/data/pass/multiarray.py,sha256=MxHax6l94yqlTVZleAqG77ILEbW6wU5osPcHzxJ85ns,1331 +numpy/typing/tests/data/pass/ndarray_conversion.py,sha256=yPgzXG6paY1uF_z-QyHYrcmrZvhX7qtvTUh7ANLseCA,1626 +numpy/typing/tests/data/pass/ndarray_misc.py,sha256=z3mucbn9fLM1gxmbUhWlp2lcrOv4zFjqZFze0caE2EA,2715 +numpy/typing/tests/data/pass/ndarray_shape_manipulation.py,sha256=37eYwMNqMLwanIW9-63hrokacnSz2K_qtPUlkdpsTjo,640 +numpy/typing/tests/data/pass/numeric.py,sha256=SdnsD5zv0wm8T2hnIylyS14ig2McSz6rG9YslckbNQ4,1490 +numpy/typing/tests/data/pass/numerictypes.py,sha256=r0_s-a0-H2MdWIn4U4P6W9RQO0V1xrDusgodHNZeIYM,750 +numpy/typing/tests/data/pass/random.py,sha256=uJCnzlsOn9hr_G1TpHLdsweJI4EdhUSEQ4dxROPjqAs,61881 +numpy/typing/tests/data/pass/scalars.py,sha256=En0adCZAwEigZrzdQ0JQwDEmrS0b-DMd1vvjkFcvwo8,3479 +numpy/typing/tests/data/pass/simple.py,sha256=HmAfCOdZBWQF211YaZFrIGisMgu5FzTELApKny08n3Y,2676 +numpy/typing/tests/data/pass/simple_py3.py,sha256=HuLrc5aphThQkLjU2_19KgGFaXwKOfSzXe0p2xMm8ZI,96 +numpy/typing/tests/data/pass/ufunc_config.py,sha256=_M8v-QWAeT1-2MkfSeAbNl_ZwyPvYfPTsLl6c1X8d_w,1204 +numpy/typing/tests/data/pass/ufunclike.py,sha256=Gve6cJ2AT3TAwOjUOQQDIUnqsRCGYq70_tv_sgODiiA,1039 +numpy/typing/tests/data/pass/ufuncs.py,sha256=xGuKuqPetUTS4io5YDHaki5nbYRu-wC29SGU32tzVIg,462 +numpy/typing/tests/data/pass/warnings_and_errors.py,sha256=Pcg-QWfY4PAhTKyehae8q6LhtbUABxa2Ye63-3h1f4w,150 +numpy/typing/tests/data/reveal/arithmetic.pyi,sha256=Ndmi_IFAl8z28RHsYTbOouf-B5FH91x_9ky-JwsdXVg,19765 +numpy/typing/tests/data/reveal/array_constructors.pyi,sha256=DcT8Z2rEpqYfjXySBejk8cGOUidUmizZGE5ZEy7r14E,10600 +numpy/typing/tests/data/reveal/arraypad.pyi,sha256=Q1pcU4B3eRsw5jsv-S0MsEfNUbp_4aMdO_o3n0rtA2A,776 +numpy/typing/tests/data/reveal/arrayprint.pyi,sha256=YyzzkL-wj4Rs-fdo3brpoaWtb5g3yk4Vn2HKu5KRo4w,876 +numpy/typing/tests/data/reveal/arraysetops.pyi,sha256=ApCFQcZzQ08zV32SJ86Xyv_7jazl3XKMmJmULtNquJ8,4155 +numpy/typing/tests/data/reveal/arrayterator.pyi,sha256=TF_1eneHoT0v9HqS9dKc5Xiv3iY3E330GR1RNcJ7s2Q,1111 +numpy/typing/tests/data/reveal/bitwise_ops.pyi,sha256=nRkyUGrBB_Es7TKyDxS_s3u2dFgBfzjocInI9Ea-J10,3919 +numpy/typing/tests/data/reveal/char.pyi,sha256=M_iTa9Pn8F7jQ1k6RN9KvbhEn00g7UYJZ5PV57ikcZM,7289 +numpy/typing/tests/data/reveal/chararray.pyi,sha256=O0EfwnKc3W1Fnx1c7Yotb1O84kVMuqJLlMBXd2duvjI,6093 +numpy/typing/tests/data/reveal/comparisons.pyi,sha256=huaf-seaF5ndTqfoaBfPtMMkOYovq7ibJl5-CRoQW7s,7468 +numpy/typing/tests/data/reveal/constants.pyi,sha256=P9vFEMkPpJ5KeUnzqPOuyHlh3zAFl9lzB4WxyB2od7A,1949 +numpy/typing/tests/data/reveal/ctypeslib.pyi,sha256=-Pk2rLEGCzz3B_y8Mu10JSVA8gPFztl5fV1dspPzqig,4727 +numpy/typing/tests/data/reveal/datasource.pyi,sha256=e8wjn60tO5EdnkBF34JrZT5XvdyW7kRWD2abtgr6qUg,671 +numpy/typing/tests/data/reveal/dtype.pyi,sha256=TKrYyxMu5IGobs0SDTIRcPuWsZ5X7zMYB4pmUlTTJxA,2872 +numpy/typing/tests/data/reveal/einsumfunc.pyi,sha256=pbtSfzIWUJRkDpe2riHBlvFlNSC3CqVM-SbYtBgX9H0,2044 +numpy/typing/tests/data/reveal/emath.pyi,sha256=-muNpWOv_niIn-zS3gUnFO4qBZAouNlVGue2x1L5Ris,2423 +numpy/typing/tests/data/reveal/false_positives.pyi,sha256=AplTmZV7TS7nivU8vegbstMN5MdMv4U0JJdZ4IeeA5M,482 +numpy/typing/tests/data/reveal/fft.pyi,sha256=ReQ9qn5frvJEy-g0RWpUGlPBntUS1cFSIu6WfPotHzE,1749 +numpy/typing/tests/data/reveal/flatiter.pyi,sha256=e1OQsVxQpgyfqMNw2puUTATl-w3swvdknlctAiWxf_E,882 +numpy/typing/tests/data/reveal/fromnumeric.pyi,sha256=PNtGQR1VmGk_xNbd0eP7k7B2oNCMBz2XOJ17-_SdE5M,12101 +numpy/typing/tests/data/reveal/getlimits.pyi,sha256=nUGOMFpWj3pMgqLy6ZbR7A4G2q7iLIl5zEFBGf-Qcfw,1592 +numpy/typing/tests/data/reveal/histograms.pyi,sha256=MxKWoa7UoJRRLim53H6OoyYfz87P3_9YUXGYPTknGVQ,1303 +numpy/typing/tests/data/reveal/index_tricks.pyi,sha256=HpD7lU7hcyDoLdZbeqskPXnX7KYwPtll7uJKYUzrlE8,3177 +numpy/typing/tests/data/reveal/lib_function_base.pyi,sha256=eSiSZUlmPXqVPKknM7GcEv76BDgj0IJRu3FXcZXpmqc,8318 +numpy/typing/tests/data/reveal/lib_polynomial.pyi,sha256=TOzOdMPDqveDv3vDKSjtq6RRvN-j_s2J7aud2ySDAB0,5986 +numpy/typing/tests/data/reveal/lib_utils.pyi,sha256=_zj7WGYGYMFXAHLK-F11aeFfDvjRvFARUjoXhbXn8V0,1049 +numpy/typing/tests/data/reveal/lib_version.pyi,sha256=UCioUeykot8-nWL6goKxZnKZxtgB4lFEi9wdN_xyF1U,672 +numpy/typing/tests/data/reveal/linalg.pyi,sha256=LPaY-RyYL7Xt3djCgNaWEgI8beI9Eo_XnvOwi6Y7-eo,4877 +numpy/typing/tests/data/reveal/matrix.pyi,sha256=ciJXsn5v2O1IZ3VEn5Ilp8-40NTQokfrOOgVXMFsvLo,2922 +numpy/typing/tests/data/reveal/memmap.pyi,sha256=A5PovMzjRp2zslF1vw3TdTQjj4Y0dIEJ__HDBV_svGM,842 +numpy/typing/tests/data/reveal/mod.pyi,sha256=-CNWft2jQGSdrO8dYRgwbl7OhL3a78Zo60JVmiY-gQI,5666 +numpy/typing/tests/data/reveal/modules.pyi,sha256=0WPq7A-aqWkJsV-IA1_7dFNCcxBacj1AWExaXbXErG4,1958 +numpy/typing/tests/data/reveal/multiarray.pyi,sha256=6MvfNKihK-oN6QwG9HFNelgheo4lnL0FCrmIF_qxdoA,5326 +numpy/typing/tests/data/reveal/nbit_base_example.pyi,sha256=DRUMGatQvQXTuovKEMF4dzazIU6it6FU53LkOEo2vNo,657 +numpy/typing/tests/data/reveal/ndarray_conversion.pyi,sha256=BfjQD8U756l4gOfY0LD47HhDRxbq0yCFfEFKvbXs7Rs,1791 +numpy/typing/tests/data/reveal/ndarray_misc.pyi,sha256=0EN-a47Msn4pZgKVdD-GrXCCmt-oxjlov5rszchBmOI,7126 +numpy/typing/tests/data/reveal/ndarray_shape_manipulation.pyi,sha256=QDQ9g6l-e73pTJp-Dosiynb-okbqi91D4KirjhIjcv4,1233 +numpy/typing/tests/data/reveal/nditer.pyi,sha256=VFXnT75BgWSUpb-dD-q5cZkfeOqsk-x9cH626g9FWT4,2021 +numpy/typing/tests/data/reveal/nested_sequence.pyi,sha256=IQyRlXduk-ZEakOtoliMLCqNgGbeg0mzZf-a-a3Gq_0,734 +numpy/typing/tests/data/reveal/npyio.pyi,sha256=YXagt2J-1suu5WXZ_si5NuJf7sHj_7NlaSLqQkam1Po,4209 +numpy/typing/tests/data/reveal/numeric.pyi,sha256=aJKnav-X45tjSFfgGD4iCetwEFcJXdNgU7valktjiCg,6160 +numpy/typing/tests/data/reveal/numerictypes.pyi,sha256=-YQRhwjBjsFJHjpGCRqzafNnKDdsmbBHbmPwccP0pLI,2487 +numpy/typing/tests/data/reveal/random.pyi,sha256=s6T074ZIpGAUqHnA-yAlozTLvt7PNBjCBqd-nGMqWGg,104091 +numpy/typing/tests/data/reveal/rec.pyi,sha256=DbRVk6lc7-3qPe-7Q26tUWpdaH9B4UVoQSYrRGJUo1Q,3858 +numpy/typing/tests/data/reveal/scalars.pyi,sha256=Qn3B3rsqSN397Jh25xs4odt2pfCQtWkoJe-e0-oX8d4,4790 +numpy/typing/tests/data/reveal/shape_base.pyi,sha256=YjiVukrK6OOydvopOaOmeAIIa0YQ2hn9_I_-FyYkHVU,2427 +numpy/typing/tests/data/reveal/stride_tricks.pyi,sha256=EBZR8gSP385nhotwJ3GH9DOUD2q5nUEYbXfhLo5xrPo,1542 +numpy/typing/tests/data/reveal/testing.pyi,sha256=_WOAj_t5SWYiqN0KG26Mza8RvaD3WAa7rFUlgksjLms,8611 +numpy/typing/tests/data/reveal/twodim_base.pyi,sha256=ZdNVo2HIJcx8iF9PA-z5W3Bs0hWM2nlVdbhLuAQlljM,3132 +numpy/typing/tests/data/reveal/type_check.pyi,sha256=yZSp50TtvPqv_PN7zmVcNOVUTUXMNYFGcguMNj25E9Y,3044 +numpy/typing/tests/data/reveal/ufunc_config.pyi,sha256=buwSvat3SVFAFl5k8TL6Mgpi32o6hHZYZ2Lpn6AHdEU,1327 +numpy/typing/tests/data/reveal/ufunclike.pyi,sha256=V_gLcZVrTXJ21VkUMwA0HyxUgA1r6OzjsdJegaKL2GE,1329 +numpy/typing/tests/data/reveal/ufuncs.pyi,sha256=VnwYr5KT_FLKfc0wV7dtNz7bNtaC9VIQt-oz56Hb5EE,2798 +numpy/typing/tests/data/reveal/warnings_and_errors.pyi,sha256=ImMlPt2PQBtX8Qf1EZFmLjNWm8fPE6IWQ_deaq_-85s,538 +numpy/typing/tests/test_isfile.py,sha256=BhKZs4-LrhFUfKjcG0yelySjE6ZITMxGIBYWGDHMRb8,864 +numpy/typing/tests/test_runtime.py,sha256=2qu8JEliITnZCBJ_QJpohacj_OQ08o73ixS2w2ooNXI,3275 +numpy/typing/tests/test_typing.py,sha256=Da1ZOFjtPh_Mvb5whpI-okBJdgLOAfJtJNyG6leGFoQ,8743 +numpy/version.py,sha256=OTLnSh0NGfWyL8VrnIj0Ndt_KZOTl1Z-kD9Cf-jRMmY,216 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/REQUESTED b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/WHEEL b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d98ef534f680b37433e9ab0f8470bdbe56c303d8 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: meson +Root-Is-Purelib: false +Tag: cp312-cp312-manylinux_2_17_x86_64 +Tag: cp312-cp312-manylinux2014_x86_64 + diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/entry_points.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..450d8ef27b9d42527fde6aced068dd7e13f2c9a3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy-1.26.4.dist-info/entry_points.txt @@ -0,0 +1,9 @@ +[array_api] +numpy = numpy.array_api + +[pyinstaller40] +hook-dirs = numpy:_pyinstaller_hooks_dir + +[console_scripts] +f2py = numpy.f2py.f2py2e:main + diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/INSTALLER b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/METADATA b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..4fae463ed9a947b135dcbdba39ae5cdfe8c932cf --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/METADATA @@ -0,0 +1,1058 @@ +Metadata-Version: 2.4 +Name: regex +Version: 2026.5.9 +Summary: Alternative regular expression module, to replace re. +Author-email: Matthew Barnett +License-Expression: Apache-2.0 AND CNRI-Python +Project-URL: Homepage, https://github.com/mrabarnett/mrab-regex +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Scientific/Engineering :: Information Analysis +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing +Classifier: Topic :: Text Processing :: General +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE.txt +Dynamic: license-file + +Introduction +------------ + +This regex implementation is backwards-compatible with the standard 're' module, but offers additional functionality. + +Python 2 +-------- + +Python 2 is no longer supported. The last release that supported Python 2 was 2021.11.10. + +PyPy +---- + +This module is targeted at CPython. It expects that all codepoints are the same width, so it won't behave properly with PyPy outside U+0000..U+007F because PyPy stores strings as UTF-8. + +Multithreading +-------------- + +The regex module releases the GIL during matching on instances of the built-in (immutable) string classes, enabling other Python threads to run concurrently. It is also possible to force the regex module to release the GIL during matching by calling the matching methods with the keyword argument ``concurrent=True``. The behaviour is undefined if the string changes during matching, so use it *only* when it is guaranteed that that won't happen. + +Unicode +------- + +This module supports Unicode 17.0.0. Full Unicode case-folding is supported. + +Flags +----- + +There are 2 kinds of flag: scoped and global. Scoped flags can apply to only part of a pattern and can be turned on or off; global flags apply to the entire pattern and can only be turned on. + +The scoped flags are: ``ASCII (?a)``, ``FULLCASE (?f)``, ``IGNORECASE (?i)``, ``LOCALE (?L)``, ``MULTILINE (?m)``, ``DOTALL (?s)``, ``UNICODE (?u)``, ``VERBOSE (?x)``, ``WORD (?w)``. + +The global flags are: ``BESTMATCH (?b)``, ``ENHANCEMATCH (?e)``, ``POSIX (?p)``, ``REVERSE (?r)``, ``VERSION0 (?V0)``, ``VERSION1 (?V1)``. + +If neither the ``ASCII``, ``LOCALE`` nor ``UNICODE`` flag is specified, it will default to ``UNICODE`` if the regex pattern is a Unicode string and ``ASCII`` if it's a bytestring. + +The ``ENHANCEMATCH`` flag makes fuzzy matching attempt to improve the fit of the next match that it finds. + +The ``BESTMATCH`` flag makes fuzzy matching search for the best match instead of the next match. + +Old vs new behaviour +-------------------- + +In order to be compatible with the re module, this module has 2 behaviours: + +* **Version 0** behaviour (old behaviour, compatible with the re module): + + Please note that the re module's behaviour may change over time, and I'll endeavour to match that behaviour in version 0. + + * Indicated by the ``VERSION0`` flag. + + * Zero-width matches are not handled correctly in the re module before Python 3.7. The behaviour in those earlier versions is: + + * ``.split`` won't split a string at a zero-width match. + + * ``.sub`` will advance by one character after a zero-width match. + + * Inline flags apply to the entire pattern, and they can't be turned off. + + * Only simple sets are supported. + + * Case-insensitive matches in Unicode use simple case-folding by default. + +* **Version 1** behaviour (new behaviour, possibly different from the re module): + + * Indicated by the ``VERSION1`` flag. + + * Zero-width matches are handled correctly. + + * Inline flags apply to the end of the group or pattern, and they can be turned off. + + * Nested sets and set operations are supported. + + * Case-insensitive matches in Unicode use full case-folding by default. + +If no version is specified, the regex module will default to ``regex.DEFAULT_VERSION``. + +Case-insensitive matches in Unicode +----------------------------------- + +The regex module supports both simple and full case-folding for case-insensitive matches in Unicode. Use of full case-folding can be turned on using the ``FULLCASE`` flag. Please note that this flag affects how the ``IGNORECASE`` flag works; the ``FULLCASE`` flag itself does not turn on case-insensitive matching. + +Version 0 behaviour: the flag is off by default. + +Version 1 behaviour: the flag is on by default. + +Nested sets and set operations +------------------------------ + +It's not possible to support both simple sets, as used in the re module, and nested sets at the same time because of a difference in the meaning of an unescaped ``"["`` in a set. + +For example, the pattern ``[[a-z]--[aeiou]]`` is treated in the version 0 behaviour (simple sets, compatible with the re module) as: + +* Set containing "[" and the letters "a" to "z" + +* Literal "--" + +* Set containing letters "a", "e", "i", "o", "u" + +* Literal "]" + +but in the version 1 behaviour (nested sets, enhanced behaviour) as: + +* Set which is: + + * Set containing the letters "a" to "z" + +* but excluding: + + * Set containing the letters "a", "e", "i", "o", "u" + +Version 0 behaviour: only simple sets are supported. + +Version 1 behaviour: nested sets and set operations are supported. + +Notes on named groups +--------------------- + +All groups have a group number, starting from 1. + +Groups with the same group name will have the same group number, and groups with a different group name will have a different group number. + +The same name can be used by more than one group, with later captures 'overwriting' earlier captures. All the captures of the group will be available from the ``captures`` method of the match object. + +Group numbers will be reused across different branches of a branch reset, eg. ``(?|(first)|(second))`` has only group 1. If groups have different group names then they will, of course, have different group numbers, eg. ``(?|(?Pfirst)|(?Psecond))`` has group 1 ("foo") and group 2 ("bar"). + +In the regex ``(\s+)(?|(?P[A-Z]+)|(\w+) (?P[0-9]+)`` there are 2 groups: + +* ``(\s+)`` is group 1. + +* ``(?P[A-Z]+)`` is group 2, also called "foo". + +* ``(\w+)`` is group 2 because of the branch reset. + +* ``(?P[0-9]+)`` is group 2 because it's called "foo". + +If you want to prevent ``(\w+)`` from being group 2, you need to name it (different name, different group number). + +Additional features +------------------- + +The issue numbers relate to the Python bug tracker, except where listed otherwise. + +Added ``\p{Horiz_Space}`` and ``\p{Vert_Space}`` (`GitHub issue 477 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``\p{Horiz_Space}`` or ``\p{H}`` matches horizontal whitespace and ``\p{Vert_Space}`` or ``\p{V}`` matches vertical whitespace. + +Added support for lookaround in conditional pattern (`Hg issue 163 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The test of a conditional pattern can be a lookaround. + +.. sourcecode:: python + + >>> regex.match(r'(?(?=\d)\d+|\w+)', '123abc') + + >>> regex.match(r'(?(?=\d)\d+|\w+)', 'abc123') + + +This is not quite the same as putting a lookaround in the first branch of a pair of alternatives. + +.. sourcecode:: python + + >>> print(regex.match(r'(?:(?=\d)\d+\b|\w+)', '123abc')) + + >>> print(regex.match(r'(?(?=\d)\d+\b|\w+)', '123abc')) + None + +In the first example, the lookaround matched, but the remainder of the first branch failed to match, and so the second branch was attempted, whereas in the second example, the lookaround matched, and the first branch failed to match, but the second branch was **not** attempted. + +Added POSIX matching (leftmost longest) (`Hg issue 150 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The POSIX standard for regex is to return the leftmost longest match. This can be turned on using the ``POSIX`` flag. + +.. sourcecode:: python + + >>> # Normal matching. + >>> regex.search(r'Mr|Mrs', 'Mrs') + + >>> regex.search(r'one(self)?(selfsufficient)?', 'oneselfsufficient') + + >>> # POSIX matching. + >>> regex.search(r'(?p)Mr|Mrs', 'Mrs') + + >>> regex.search(r'(?p)one(self)?(selfsufficient)?', 'oneselfsufficient') + + +Note that it will take longer to find matches because when it finds a match at a certain position, it won't return that immediately, but will keep looking to see if there's another longer match there. + +Added ``(?(DEFINE)...)`` (`Hg issue 152 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If there's no group called "DEFINE", then ... will be ignored except that any groups defined within it can be called and that the normal rules for numbering groups still apply. + +.. sourcecode:: python + + >>> regex.search(r'(?(DEFINE)(?P\d+)(?P\w+))(?&quant) (?&item)', '5 elephants') + + +Added ``(*PRUNE)``, ``(*SKIP)`` and ``(*FAIL)`` (`Hg issue 153 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``(*PRUNE)`` discards the backtracking info up to that point. When used in an atomic group or a lookaround, it won't affect the enclosing pattern. + +``(*SKIP)`` is similar to ``(*PRUNE)``, except that it also sets where in the text the next attempt to match will start. When used in an atomic group or a lookaround, it won't affect the enclosing pattern. + +``(*FAIL)`` causes immediate backtracking. ``(*F)`` is a permitted abbreviation. + +Added ``\K`` (`Hg issue 151 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Keeps the part of the entire match after the position where ``\K`` occurred; the part before it is discarded. + +It does not affect what groups return. + +.. sourcecode:: python + + >>> m = regex.search(r'(\w\w\K\w\w\w)', 'abcdef') + >>> m[0] + 'cde' + >>> m[1] + 'abcde' + >>> + >>> m = regex.search(r'(?r)(\w\w\K\w\w\w)', 'abcdef') + >>> m[0] + 'bc' + >>> m[1] + 'bcdef' + +Added capture subscripting for ``expandf`` and ``subf``/``subfn`` (`Hg issue 133 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use subscripting to get the captures of a repeated group. + +.. sourcecode:: python + + >>> m = regex.match(r"(\w)+", "abc") + >>> m.expandf("{1}") + 'c' + >>> m.expandf("{1[0]} {1[1]} {1[2]}") + 'a b c' + >>> m.expandf("{1[-1]} {1[-2]} {1[-3]}") + 'c b a' + >>> + >>> m = regex.match(r"(?P\w)+", "abc") + >>> m.expandf("{letter}") + 'c' + >>> m.expandf("{letter[0]} {letter[1]} {letter[2]}") + 'a b c' + >>> m.expandf("{letter[-1]} {letter[-2]} {letter[-3]}") + 'c b a' + +Added support for referring to a group by number using ``(?P=...)`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is in addition to the existing ``\g<...>``. + +Fixed the handling of locale-sensitive regexes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``LOCALE`` flag is intended for legacy code and has limited support. You're still recommended to use Unicode instead. + +Added partial matches (`Hg issue 102 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A partial match is one that matches up to the end of string, but that string has been truncated and you want to know whether a complete match could be possible if the string had not been truncated. + +Partial matches are supported by ``match``, ``search``, ``fullmatch`` and ``finditer`` with the ``partial`` keyword argument. + +Match objects have a ``partial`` attribute, which is ``True`` if it's a partial match. + +For example, if you wanted a user to enter a 4-digit number and check it character by character as it was being entered: + +.. sourcecode:: python + + >>> pattern = regex.compile(r'\d{4}') + + >>> # Initially, nothing has been entered: + >>> print(pattern.fullmatch('', partial=True)) + + + >>> # An empty string is OK, but it's only a partial match. + >>> # The user enters a letter: + >>> print(pattern.fullmatch('a', partial=True)) + None + >>> # It'll never match. + + >>> # The user deletes that and enters a digit: + >>> print(pattern.fullmatch('1', partial=True)) + + >>> # It matches this far, but it's only a partial match. + + >>> # The user enters 2 more digits: + >>> print(pattern.fullmatch('123', partial=True)) + + >>> # It matches this far, but it's only a partial match. + + >>> # The user enters another digit: + >>> print(pattern.fullmatch('1234', partial=True)) + + >>> # It's a complete match. + + >>> # If the user enters another digit: + >>> print(pattern.fullmatch('12345', partial=True)) + None + >>> # It's no longer a match. + + >>> # This is a partial match: + >>> pattern.match('123', partial=True).partial + True + + >>> # This is a complete match: + >>> pattern.match('1233', partial=True).partial + False + +``*`` operator not working correctly with sub() (`Hg issue 106 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes it's not clear how zero-width matches should be handled. For example, should ``.*`` match 0 characters directly after matching >0 characters? + +.. sourcecode:: python + + >>> regex.sub('.*', 'x', 'test') + 'xx' + >>> regex.sub('.*?', '|', 'test') + '|||||||||' + +Added ``capturesdict`` (`Hg issue 86 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``capturesdict`` is a combination of ``groupdict`` and ``captures``: + +``groupdict`` returns a dict of the named groups and the last capture of those groups. + +``captures`` returns a list of all the captures of a group + +``capturesdict`` returns a dict of the named groups and lists of all the captures of those groups. + +.. sourcecode:: python + + >>> m = regex.match(r"(?:(?P\w+) (?P\d+)\n)+", "one 1\ntwo 2\nthree 3\n") + >>> m.groupdict() + {'word': 'three', 'digits': '3'} + >>> m.captures("word") + ['one', 'two', 'three'] + >>> m.captures("digits") + ['1', '2', '3'] + >>> m.capturesdict() + {'word': ['one', 'two', 'three'], 'digits': ['1', '2', '3']} + +Added ``allcaptures`` and ``allspans`` (`Git issue 474 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``allcaptures`` returns a list of all the captures of all the groups. + +``allspans`` returns a list of all the spans of the all captures of all the groups. + +.. sourcecode:: python + + >>> m = regex.match(r"(?:(?P\w+) (?P\d+)\n)+", "one 1\ntwo 2\nthree 3\n") + >>> m.allcaptures() + (['one 1\ntwo 2\nthree 3\n'], ['one', 'two', 'three'], ['1', '2', '3']) + >>> m.allspans() + ([(0, 20)], [(0, 3), (6, 9), (12, 17)], [(4, 5), (10, 11), (18, 19)]) + +Allow duplicate names of groups (`Hg issue 87 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Group names can be duplicated. + +.. sourcecode:: python + + >>> # With optional groups: + >>> + >>> # Both groups capture, the second capture 'overwriting' the first. + >>> m = regex.match(r"(?P\w+)? or (?P\w+)?", "first or second") + >>> m.group("item") + 'second' + >>> m.captures("item") + ['first', 'second'] + >>> # Only the second group captures. + >>> m = regex.match(r"(?P\w+)? or (?P\w+)?", " or second") + >>> m.group("item") + 'second' + >>> m.captures("item") + ['second'] + >>> # Only the first group captures. + >>> m = regex.match(r"(?P\w+)? or (?P\w+)?", "first or ") + >>> m.group("item") + 'first' + >>> m.captures("item") + ['first'] + >>> + >>> # With mandatory groups: + >>> + >>> # Both groups capture, the second capture 'overwriting' the first. + >>> m = regex.match(r"(?P\w*) or (?P\w*)?", "first or second") + >>> m.group("item") + 'second' + >>> m.captures("item") + ['first', 'second'] + >>> # Again, both groups capture, the second capture 'overwriting' the first. + >>> m = regex.match(r"(?P\w*) or (?P\w*)", " or second") + >>> m.group("item") + 'second' + >>> m.captures("item") + ['', 'second'] + >>> # And yet again, both groups capture, the second capture 'overwriting' the first. + >>> m = regex.match(r"(?P\w*) or (?P\w*)", "first or ") + >>> m.group("item") + '' + >>> m.captures("item") + ['first', ''] + +Added ``fullmatch`` (`issue #16203 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``fullmatch`` behaves like ``match``, except that it must match all of the string. + +.. sourcecode:: python + + >>> print(regex.fullmatch(r"abc", "abc").span()) + (0, 3) + >>> print(regex.fullmatch(r"abc", "abcx")) + None + >>> print(regex.fullmatch(r"abc", "abcx", endpos=3).span()) + (0, 3) + >>> print(regex.fullmatch(r"abc", "xabcy", pos=1, endpos=4).span()) + (1, 4) + >>> + >>> regex.match(r"a.*?", "abcd").group(0) + 'a' + >>> regex.fullmatch(r"a.*?", "abcd").group(0) + 'abcd' + +Added ``subf`` and ``subfn`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``subf`` and ``subfn`` are alternatives to ``sub`` and ``subn`` respectively. When passed a replacement string, they treat it as a format string. + +.. sourcecode:: python + + >>> regex.subf(r"(\w+) (\w+)", "{0} => {2} {1}", "foo bar") + 'foo bar => bar foo' + >>> regex.subf(r"(?P\w+) (?P\w+)", "{word2} {word1}", "foo bar") + 'bar foo' + +Added ``expandf`` to match object +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``expandf`` is an alternative to ``expand``. When passed a replacement string, it treats it as a format string. + +.. sourcecode:: python + + >>> m = regex.match(r"(\w+) (\w+)", "foo bar") + >>> m.expandf("{0} => {2} {1}") + 'foo bar => bar foo' + >>> + >>> m = regex.match(r"(?P\w+) (?P\w+)", "foo bar") + >>> m.expandf("{word2} {word1}") + 'bar foo' + +Detach searched string +^^^^^^^^^^^^^^^^^^^^^^ + +A match object contains a reference to the string that was searched, via its ``string`` attribute. The ``detach_string`` method will 'detach' that string, making it available for garbage collection, which might save valuable memory if that string is very large. + +.. sourcecode:: python + + >>> m = regex.search(r"\w+", "Hello world") + >>> print(m.group()) + Hello + >>> print(m.string) + Hello world + >>> m.detach_string() + >>> print(m.group()) + Hello + >>> print(m.string) + None + +Recursive patterns (`Hg issue 27 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Recursive and repeated patterns are supported. + +``(?R)`` or ``(?0)`` tries to match the entire regex recursively. ``(?1)``, ``(?2)``, etc, try to match the relevant group. + +``(?&name)`` tries to match the named group. + +.. sourcecode:: python + + >>> regex.match(r"(Tarzan|Jane) loves (?1)", "Tarzan loves Jane").groups() + ('Tarzan',) + >>> regex.match(r"(Tarzan|Jane) loves (?1)", "Jane loves Tarzan").groups() + ('Jane',) + + >>> m = regex.search(r"(\w)(?:(?R)|(\w?))\1", "kayak") + >>> m.group(0, 1, 2) + ('kayak', 'k', None) + +The first two examples show how the subpattern within the group is reused, but is _not_ itself a group. In other words, ``"(Tarzan|Jane) loves (?1)"`` is equivalent to ``"(Tarzan|Jane) loves (?:Tarzan|Jane)"``. + +It's possible to backtrack into a recursed or repeated group. + +You can't call a group if there is more than one group with that group name or group number (``"ambiguous group reference"``). + +The alternative forms ``(?P>name)`` and ``(?P&name)`` are also supported. + +Full Unicode case-folding is supported +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In version 1 behaviour, the regex module uses full case-folding when performing case-insensitive matches in Unicode. + +.. sourcecode:: python + + >>> regex.match(r"(?iV1)strasse", "stra\N{LATIN SMALL LETTER SHARP S}e").span() + (0, 6) + >>> regex.match(r"(?iV1)stra\N{LATIN SMALL LETTER SHARP S}e", "STRASSE").span() + (0, 7) + +In version 0 behaviour, it uses simple case-folding for backward compatibility with the re module. + +Approximate "fuzzy" matching (`Hg issue 12 `_, `Hg issue 41 `_, `Hg issue 109 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Regex usually attempts an exact match, but sometimes an approximate, or "fuzzy", match is needed, for those cases where the text being searched may contain errors in the form of inserted, deleted or substituted characters. + +A fuzzy regex specifies which types of errors are permitted, and, optionally, either the minimum and maximum or only the maximum permitted number of each type. (You cannot specify only a minimum.) + +The 3 types of error are: + +* Insertion, indicated by "i" + +* Deletion, indicated by "d" + +* Substitution, indicated by "s" + +In addition, "e" indicates any type of error. + +The fuzziness of a regex item is specified between "{" and "}" after the item. + +Examples: + +* ``foo`` match "foo" exactly + +* ``(?:foo){i}`` match "foo", permitting insertions + +* ``(?:foo){d}`` match "foo", permitting deletions + +* ``(?:foo){s}`` match "foo", permitting substitutions + +* ``(?:foo){i,s}`` match "foo", permitting insertions and substitutions + +* ``(?:foo){e}`` match "foo", permitting errors + +If a certain type of error is specified, then any type not specified will **not** be permitted. + +In the following examples I'll omit the item and write only the fuzziness: + +* ``{d<=3}`` permit at most 3 deletions, but no other types + +* ``{i<=1,s<=2}`` permit at most 1 insertion and at most 2 substitutions, but no deletions + +* ``{1<=e<=3}`` permit at least 1 and at most 3 errors + +* ``{i<=2,d<=2,e<=3}`` permit at most 2 insertions, at most 2 deletions, at most 3 errors in total, but no substitutions + +It's also possible to state the costs of each type of error and the maximum permitted total cost. + +Examples: + +* ``{2i+2d+1s<=4}`` each insertion costs 2, each deletion costs 2, each substitution costs 1, the total cost must not exceed 4 + +* ``{i<=1,d<=1,s<=1,2i+2d+1s<=4}`` at most 1 insertion, at most 1 deletion, at most 1 substitution; each insertion costs 2, each deletion costs 2, each substitution costs 1, the total cost must not exceed 4 + +You can also use "<" instead of "<=" if you want an exclusive minimum or maximum. + +You can add a test to perform on a character that's substituted or inserted. + +Examples: + +* ``{s<=2:[a-z]}`` at most 2 substitutions, which must be in the character set ``[a-z]``. + +* ``{s<=2,i<=3:\d}`` at most 2 substitutions, at most 3 insertions, which must be digits. + +By default, fuzzy matching searches for the first match that meets the given constraints. The ``ENHANCEMATCH`` flag will cause it to attempt to improve the fit (i.e. reduce the number of errors) of the match that it has found. + +The ``BESTMATCH`` flag will make it search for the best match instead. + +Further examples to note: + +* ``regex.search("(dog){e}", "cat and dog")[1]`` returns ``"cat"`` because that matches ``"dog"`` with 3 errors (an unlimited number of errors is permitted). + +* ``regex.search("(dog){e<=1}", "cat and dog")[1]`` returns ``" dog"`` (with a leading space) because that matches ``"dog"`` with 1 error, which is within the limit. + +* ``regex.search("(?e)(dog){e<=1}", "cat and dog")[1]`` returns ``"dog"`` (without a leading space) because the fuzzy search matches ``" dog"`` with 1 error, which is within the limit, and the ``(?e)`` then it attempts a better fit. + +In the first two examples there are perfect matches later in the string, but in neither case is it the first possible match. + +The match object has an attribute ``fuzzy_counts`` which gives the total number of substitutions, insertions and deletions. + +.. sourcecode:: python + + >>> # A 'raw' fuzzy match: + >>> regex.fullmatch(r"(?:cats|cat){e<=1}", "cat").fuzzy_counts + (0, 0, 1) + >>> # 0 substitutions, 0 insertions, 1 deletion. + + >>> # A better match might be possible if the ENHANCEMATCH flag used: + >>> regex.fullmatch(r"(?e)(?:cats|cat){e<=1}", "cat").fuzzy_counts + (0, 0, 0) + >>> # 0 substitutions, 0 insertions, 0 deletions. + +The match object also has an attribute ``fuzzy_changes`` which gives a tuple of the positions of the substitutions, insertions and deletions. + +.. sourcecode:: python + + >>> m = regex.search('(fuu){i<=2,d<=2,e<=5}', 'anaconda foo bar') + >>> m + + >>> m.fuzzy_changes + ([], [7, 8], [10, 11]) + +What this means is that if the matched part of the string had been: + +.. sourcecode:: python + + 'anacondfuuoo bar' + +it would've been an exact match. + +However, there were insertions at positions 7 and 8: + +.. sourcecode:: python + + 'anaconda fuuoo bar' + ^^ + +and deletions at positions 10 and 11: + +.. sourcecode:: python + + 'anaconda f~~oo bar' + ^^ + +So the actual string was: + +.. sourcecode:: python + + 'anaconda foo bar' + +Named lists ``\L`` (`Hg issue 11 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are occasions where you may want to include a list (actually, a set) of options in a regex. + +One way is to build the pattern like this: + +.. sourcecode:: python + + >>> p = regex.compile(r"first|second|third|fourth|fifth") + +but if the list is large, parsing the resulting regex can take considerable time, and care must also be taken that the strings are properly escaped and properly ordered, for example, "cats" before "cat". + +The new alternative is to use a named list: + +.. sourcecode:: python + + >>> option_set = ["first", "second", "third", "fourth", "fifth"] + >>> p = regex.compile(r"\L", options=option_set) + +The order of the items is irrelevant, they are treated as a set. The named lists are available as the ``.named_lists`` attribute of the pattern object : + +.. sourcecode:: python + + >>> print(p.named_lists) + {'options': frozenset({'third', 'first', 'fifth', 'fourth', 'second'})} + +If there are any unused keyword arguments, ``ValueError`` will be raised unless you tell it otherwise: + +.. sourcecode:: python + + >>> option_set = ["first", "second", "third", "fourth", "fifth"] + >>> p = regex.compile(r"\L", options=option_set, other_options=[]) + Traceback (most recent call last): + File "", line 1, in + File "C:\Python310\lib\site-packages\regex\regex.py", line 353, in compile + return _compile(pattern, flags, ignore_unused, kwargs, cache_pattern) + File "C:\Python310\lib\site-packages\regex\regex.py", line 500, in _compile + complain_unused_args() + File "C:\Python310\lib\site-packages\regex\regex.py", line 483, in complain_unused_args + raise ValueError('unused keyword argument {!a}'.format(any_one)) + ValueError: unused keyword argument 'other_options' + >>> p = regex.compile(r"\L", options=option_set, other_options=[], ignore_unused=True) + >>> p = regex.compile(r"\L", options=option_set, other_options=[], ignore_unused=False) + Traceback (most recent call last): + File "", line 1, in + File "C:\Python310\lib\site-packages\regex\regex.py", line 353, in compile + return _compile(pattern, flags, ignore_unused, kwargs, cache_pattern) + File "C:\Python310\lib\site-packages\regex\regex.py", line 500, in _compile + complain_unused_args() + File "C:\Python310\lib\site-packages\regex\regex.py", line 483, in complain_unused_args + raise ValueError('unused keyword argument {!a}'.format(any_one)) + ValueError: unused keyword argument 'other_options' + >>> + +Start and end of word +^^^^^^^^^^^^^^^^^^^^^ + +``\m`` matches at the start of a word. + +``\M`` matches at the end of a word. + +Compare with ``\b``, which matches at the start or end of a word. + +Unicode line separators +^^^^^^^^^^^^^^^^^^^^^^^ + +Normally the only line separator is ``\n`` (``\x0A``), but if the ``WORD`` flag is turned on then the line separators are ``\x0D\x0A``, ``\x0A``, ``\x0B``, ``\x0C`` and ``\x0D``, plus ``\x85``, ``\u2028`` and ``\u2029`` when working with Unicode. + +This affects the regex dot ``"."``, which, with the ``DOTALL`` flag turned off, matches any character except a line separator. It also affects the line anchors ``^`` and ``$`` (in multiline mode). + +Set operators +^^^^^^^^^^^^^ + +**Version 1 behaviour only** + +Set operators have been added, and a set ``[...]`` can include nested sets. + +The operators, in order of increasing precedence, are: + +* ``||`` for union ("x||y" means "x or y") + +* ``~~`` (double tilde) for symmetric difference ("x~~y" means "x or y, but not both") + +* ``&&`` for intersection ("x&&y" means "x and y") + +* ``--`` (double dash) for difference ("x--y" means "x but not y") + +Implicit union, ie, simple juxtaposition like in ``[ab]``, has the highest precedence. Thus, ``[ab&&cd]`` is the same as ``[[a||b]&&[c||d]]``. + +Examples: + +* ``[ab]`` # Set containing 'a' and 'b' + +* ``[a-z]`` # Set containing 'a' .. 'z' + +* ``[[a-z]--[qw]]`` # Set containing 'a' .. 'z', but not 'q' or 'w' + +* ``[a-z--qw]`` # Same as above + +* ``[\p{L}--QW]`` # Set containing all letters except 'Q' and 'W' + +* ``[\p{N}--[0-9]]`` # Set containing all numbers except '0' .. '9' + +* ``[\p{ASCII}&&\p{Letter}]`` # Set containing all characters which are ASCII and letter + +regex.escape (`issue #2650 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +regex.escape has an additional keyword parameter ``special_only``. When True, only 'special' regex characters, such as '?', are escaped. + +.. sourcecode:: python + + >>> regex.escape("foo!?", special_only=False) + 'foo\\!\\?' + >>> regex.escape("foo!?", special_only=True) + 'foo!\\?' + +regex.escape (`Hg issue 249 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +regex.escape has an additional keyword parameter ``literal_spaces``. When True, spaces are not escaped. + +.. sourcecode:: python + + >>> regex.escape("foo bar!?", literal_spaces=False) + 'foo\\ bar!\\?' + >>> regex.escape("foo bar!?", literal_spaces=True) + 'foo bar!\\?' + +Repeated captures (`issue #7132 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A match object has additional methods which return information on all the successful matches of a repeated group. These methods are: + +* ``matchobject.captures([group1, ...])`` + + * Returns a list of the strings matched in a group or groups. Compare with ``matchobject.group([group1, ...])``. + +* ``matchobject.starts([group])`` + + * Returns a list of the start positions. Compare with ``matchobject.start([group])``. + +* ``matchobject.ends([group])`` + + * Returns a list of the end positions. Compare with ``matchobject.end([group])``. + +* ``matchobject.spans([group])`` + + * Returns a list of the spans. Compare with ``matchobject.span([group])``. + +.. sourcecode:: python + + >>> m = regex.search(r"(\w{3})+", "123456789") + >>> m.group(1) + '789' + >>> m.captures(1) + ['123', '456', '789'] + >>> m.start(1) + 6 + >>> m.starts(1) + [0, 3, 6] + >>> m.end(1) + 9 + >>> m.ends(1) + [3, 6, 9] + >>> m.span(1) + (6, 9) + >>> m.spans(1) + [(0, 3), (3, 6), (6, 9)] + +Atomic grouping ``(?>...)`` (`issue #433030 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the following pattern subsequently fails, then the subpattern as a whole will fail. + +Possessive quantifiers +^^^^^^^^^^^^^^^^^^^^^^ + +``(?:...)?+`` ; ``(?:...)*+`` ; ``(?:...)++`` ; ``(?:...){min,max}+`` + +The subpattern is matched up to 'max' times. If the following pattern subsequently fails, then all the repeated subpatterns will fail as a whole. For example, ``(?:...)++`` is equivalent to ``(?>(?:...)+)``. + +Scoped flags (`issue #433028 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``(?flags-flags:...)`` + +The flags will apply only to the subpattern. Flags can be turned on or off. + +Definition of 'word' character (`issue #1693050 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The definition of a 'word' character has been expanded for Unicode. It conforms to the Unicode specification at ``http://www.unicode.org/reports/tr29/``. + +Variable-length lookbehind +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A lookbehind can match a variable-length string. + +Flags argument for regex.split, regex.sub and regex.subn (`issue #3482 `_) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``regex.split``, ``regex.sub`` and ``regex.subn`` support a 'flags' argument. + +Pos and endpos arguments for regex.sub and regex.subn +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``regex.sub`` and ``regex.subn`` support 'pos' and 'endpos' arguments. + +'Overlapped' argument for regex.findall and regex.finditer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``regex.findall`` and ``regex.finditer`` support an 'overlapped' flag which permits overlapped matches. + +Splititer +^^^^^^^^^ + +``regex.splititer`` has been added. It's a generator equivalent of ``regex.split``. + +Subscripting match objects for groups +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A match object accepts access to the groups via subscripting and slicing: + +.. sourcecode:: python + + >>> m = regex.search(r"(?P.*?)(?P\d+)(?P.*)", "pqr123stu") + >>> print(m["before"]) + pqr + >>> print(len(m)) + 4 + >>> print(m[:]) + ('pqr123stu', 'pqr', '123', 'stu') + +Named groups +^^^^^^^^^^^^ + +Groups can be named with ``(?...)`` as well as the existing ``(?P...)``. + +Group references +^^^^^^^^^^^^^^^^ + +Groups can be referenced within a pattern with ``\g``. This also allows there to be more than 99 groups. + +Named characters ``\N{name}`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Named characters are supported. Note that only those known by Python's Unicode database will be recognised. + +Unicode codepoint properties, including scripts and blocks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``\p{property=value}``; ``\P{property=value}``; ``\p{value}`` ; ``\P{value}`` + +Many Unicode properties are supported, including blocks and scripts. ``\p{property=value}`` or ``\p{property:value}`` matches a character whose property ``property`` has value ``value``. The inverse of ``\p{property=value}`` is ``\P{property=value}`` or ``\p{^property=value}``. + +If the short form ``\p{value}`` is used, the properties are checked in the order: ``General_Category``, ``Script``, ``Block``, binary property: + +* ``Latin``, the 'Latin' script (``Script=Latin``). + +* ``BasicLatin``, the 'BasicLatin' block (``Block=BasicLatin``). + +* ``Alphabetic``, the 'Alphabetic' binary property (``Alphabetic=Yes``). + +A short form starting with ``Is`` indicates a script or binary property: + +* ``IsLatin``, the 'Latin' script (``Script=Latin``). + +* ``IsAlphabetic``, the 'Alphabetic' binary property (``Alphabetic=Yes``). + +A short form starting with ``In`` indicates a block property: + +* ``InBasicLatin``, the 'BasicLatin' block (``Block=BasicLatin``). + +POSIX character classes +^^^^^^^^^^^^^^^^^^^^^^^ + +``[[:alpha:]]``; ``[[:^alpha:]]`` + +POSIX character classes are supported. These are normally treated as an alternative form of ``\p{...}``. + +The exceptions are ``alnum``, ``digit``, ``punct`` and ``xdigit``, whose definitions are different from those of Unicode. + +``[[:alnum:]]`` is equivalent to ``\p{posix_alnum}``. + +``[[:digit:]]`` is equivalent to ``\p{posix_digit}``. + +``[[:punct:]]`` is equivalent to ``\p{posix_punct}``. + +``[[:xdigit:]]`` is equivalent to ``\p{posix_xdigit}``. + +Search anchor ``\G`` +^^^^^^^^^^^^^^^^^^^^ + +A search anchor has been added. It matches at the position where each search started/continued and can be used for contiguous matches or in negative variable-length lookbehinds to limit how far back the lookbehind goes: + +.. sourcecode:: python + + >>> regex.findall(r"\w{2}", "abcd ef") + ['ab', 'cd', 'ef'] + >>> regex.findall(r"\G\w{2}", "abcd ef") + ['ab', 'cd'] + +* The search starts at position 0 and matches 'ab'. + +* The search continues at position 2 and matches 'cd'. + +* The search continues at position 4 and fails to match any letters. + +* The anchor stops the search start position from being advanced, so there are no more results. + +Reverse searching +^^^^^^^^^^^^^^^^^ + +Searches can also work backwards: + +.. sourcecode:: python + + >>> regex.findall(r".", "abc") + ['a', 'b', 'c'] + >>> regex.findall(r"(?r).", "abc") + ['c', 'b', 'a'] + +Note that the result of a reverse search is not necessarily the reverse of a forward search: + +.. sourcecode:: python + + >>> regex.findall(r"..", "abcde") + ['ab', 'cd'] + >>> regex.findall(r"(?r)..", "abcde") + ['de', 'bc'] + +Matching a single grapheme ``\X`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The grapheme matcher is supported. It conforms to the Unicode specification at ``http://www.unicode.org/reports/tr29/``. + +Branch reset ``(?|...|...)`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Group numbers will be reused across the alternatives, but groups with different names will have different group numbers. + +.. sourcecode:: python + + >>> regex.match(r"(?|(first)|(second))", "first").groups() + ('first',) + >>> regex.match(r"(?|(first)|(second))", "second").groups() + ('second',) + +Note that there is only one group. + +Default Unicode word boundary +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``WORD`` flag changes the definition of a 'word boundary' to that of a default Unicode word boundary. This applies to ``\b`` and ``\B``. + +Timeout +^^^^^^^ + +The matching methods and functions support timeouts. The timeout (in seconds) applies to the entire operation: + +.. sourcecode:: python + + >>> from time import sleep + >>> + >>> def fast_replace(m): + ... return 'X' + ... + >>> def slow_replace(m): + ... sleep(0.5) + ... return 'X' + ... + >>> regex.sub(r'[a-z]', fast_replace, 'abcde', timeout=2) + 'XXXXX' + >>> regex.sub(r'[a-z]', slow_replace, 'abcde', timeout=2) + Traceback (most recent call last): + File "", line 1, in + File "C:\Python310\lib\site-packages\regex\regex.py", line 278, in sub + return pat.sub(repl, string, count, pos, endpos, concurrent, timeout) + TimeoutError: regex timed out diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/RECORD b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..1dae8db76b42461662f1037e0bc3c4b2eceb99cb --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/RECORD @@ -0,0 +1,12 @@ +regex-2026.5.9.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +regex-2026.5.9.dist-info/METADATA,sha256=Rj0SxHVZWOqnoR0K7S0OXmKfZ9A-9IRnfuPHLXDsz4E,40417 +regex-2026.5.9.dist-info/RECORD,, +regex-2026.5.9.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +regex-2026.5.9.dist-info/WHEEL,sha256=Tc3fF66yn9Kh-hkUUsdKQyuB9Lw0CDoeANnEbSVc3f4,190 +regex-2026.5.9.dist-info/licenses/LICENSE.txt,sha256=v_Ve9M3MjBTOJZ-OirYOJkQYRA1jNfTcE4Jz-9UGFE0,11584 +regex-2026.5.9.dist-info/top_level.txt,sha256=aQmiDMhNTF26cCK4_7D-qaVvhbxClG0wyCTnEhkzYBs,6 +regex/__init__.py,sha256=A5k0rm8Ln7HKsfG-8sEWYehek8zCKEe6sQW5QGUAmSU,75 +regex/_main.py,sha256=AoKU8VjL0RCcJHYBnmzRU1gCrmipmhhHcBAgaD2PWQA,33343 +regex/_regex.cpython-312-x86_64-linux-gnu.so,sha256=bAXnZf4jlgypfRi6Urcto9H1TPiz62LruOiUjtJNt4s,2570544 +regex/_regex_core.py,sha256=qDeLnme3R-VJDImGVqS6I2cq-jdy0-w86Xal8jSNDHo,147374 +regex/tests/test_regex.py,sha256=iE4XOfiEpWrGKDUZISDxlxNi-Y0rX8KPH8yAwpMTLbI,225809 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/REQUESTED b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/WHEEL b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..0493eaf8b2602d53e7eabac2ac142e2e951bac28 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/WHEEL @@ -0,0 +1,7 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: false +Tag: cp312-cp312-manylinux_2_17_x86_64 +Tag: cp312-cp312-manylinux2014_x86_64 +Tag: cp312-cp312-manylinux_2_28_x86_64 + diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/licenses/LICENSE.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..99c19cf844141395a87c34325da3a537f1e95815 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/licenses/LICENSE.txt @@ -0,0 +1,208 @@ +This work was derived from the 're' module of CPython 2.6 and CPython 3.1, +copyright (c) 1998-2001 by Secret Labs AB and licensed under CNRI's Python 1.6 +license. + +All additions and alterations are licensed under the Apache 2.0 License. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Matthew Barnett + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/top_level.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..4f9256d62325c75de027d1cd48f1ff520117413e --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/regex-2026.5.9.dist-info/top_level.txt @@ -0,0 +1 @@ +regex diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/llama4/modeling_llama4.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/llama4/modeling_llama4.py new file mode 100644 index 0000000000000000000000000000000000000000..e064e399e6c0dfcfc3efe5eae0b6fa3d25d19b47 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/llama4/modeling_llama4.py @@ -0,0 +1,1412 @@ +# Copyright 2025 The LLAMA4 and HuggingFace Inc. team. All rights reserved. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from collections.abc import Callable +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers.models.llama4.configuration_llama4 import Llama4VisionConfig + +from ... import initialization as init +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub +from ...masking_utils import create_causal_mask, create_chunked_causal_mask +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPast, + BaseModelOutputWithPooling, + CausalLMOutputWithPast, + ModelOutput, +) +from ...modeling_rope_utils import ( + ROPE_INIT_FUNCTIONS, + dynamic_rope_update, +) +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_compilable_check +from ...utils.generic import maybe_autocast, merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_llama4 import Llama4Config, Llama4TextConfig + + +logger = logging.get_logger(__name__) + + +class Llama4TextExperts(nn.Module): + def __init__(self, config: Llama4TextConfig): + super().__init__() + self.num_experts = config.num_local_experts + self.intermediate_size = config.intermediate_size + self.hidden_size = config.hidden_size + self.expert_dim = self.intermediate_size + self.gate_up_proj = nn.Parameter(torch.zeros(self.num_experts, self.hidden_size, 2 * self.expert_dim)) + self.down_proj = nn.Parameter(torch.empty((self.num_experts, self.expert_dim, self.hidden_size))) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """ + This should really not be run on a single machine, as we are reaching compute bound: + - the inputs are expected to be "sorted" per expert already. + - the weights are viewed with another dim, to match num_expert, 1, shape * num_tokens, shape + + Args: + hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) + selected_experts (torch.Tensor): (batch_size * token_num, top_k) + routing_weights (torch.Tensor): (batch_size * token_num, top_k) + Returns: + torch.Tensor + """ + hidden_states = hidden_states.view(self.gate_up_proj.shape[0], -1, self.hidden_size) + gate_up = torch.bmm(hidden_states, self.gate_up_proj) + gate, up = gate_up.chunk(2, dim=-1) # not supported for DTensors + next_states = torch.bmm((up * self.act_fn(gate)), self.down_proj) + next_states = next_states.view(-1, self.hidden_size) + return next_states + + +# Phi3MLP +class Llama4TextMLP(nn.Module): + def __init__(self, config, intermediate_size=None): + super().__init__() + + if intermediate_size is None: + intermediate_size = config.intermediate_size + + self.config = config + self.gate_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False) + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.activation_fn(self.gate_proj(x)) * self.up_proj(x) + return self.down_proj(down_proj) + + +class Llama4TextL2Norm(torch.nn.Module): + def __init__(self, eps: float = 1e-6): + super().__init__() + self.eps = eps + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + return self._norm(x.float()).type_as(x) + + def extra_repr(self): + return f"eps={self.eps}" + + +class Llama4TextRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-5): + """ + Llama4RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + return output * self.weight + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.eps}" + + +class Llama4Router(nn.Linear): + def __init__(self, config): + super().__init__(config.hidden_size, config.num_local_experts, bias=False) + self.num_experts = config.num_local_experts + self.top_k = config.num_experts_per_tok + + def forward(self, hidden_states): + router_logits = super().forward(hidden_states) + router_top_value, router_indices = torch.topk(router_logits, self.top_k, dim=1) + router_scores = torch.full_like(router_logits, float("-inf")).scatter_(1, router_indices, router_top_value) + router_scores = torch.nn.functional.sigmoid(router_scores.float()).to(router_scores.dtype) + return router_scores, router_logits + + +@use_kernel_forward_from_hub("Llama4TextMoe") +class Llama4TextMoe(nn.Module): + def __init__(self, config): + super().__init__() + self.top_k = config.num_experts_per_tok + self.hidden_dim = config.hidden_size + self.num_experts = config.num_local_experts + self.experts = Llama4TextExperts(config) + self.router = Llama4Router(config) + self.shared_expert = Llama4TextMLP(config) + + def forward(self, hidden_states): + hidden_states = hidden_states.reshape(-1, self.hidden_dim) + router_scores, router_logits = self.router(hidden_states) + routed_in = hidden_states.repeat(router_scores.shape[1], 1) + routed_in = routed_in * router_scores.transpose(0, 1).reshape(-1, 1) + routed_out = self.experts(routed_in) + out = self.shared_expert(hidden_states) + out.add_(routed_out.reshape(router_scores.shape[1], -1, routed_out.shape[-1]).sum(dim=0)) + return out, router_logits + + +# Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Llama4Text +class Llama4TextRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + # Ignore copy + def __init__(self, config: Llama4TextConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters( + config: Llama4TextConfig | None = None, + device: Optional["torch.device"] = None, + seq_len: int | None = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + # Ignore copy + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with maybe_autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.to(x.device) @ position_ids_expanded).transpose(1, 2) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # Convert to complex representation + freqs_cis = freqs_cis * self.attention_scaling + + return freqs_cis + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: torch.Tensor, + freqs_cis: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) + xq_out = torch.view_as_real(xq_ * freqs_cis[:, :, None, :]).flatten(3) + xk_out = torch.view_as_real(xk_ * freqs_cis[:, :, None, :]).flatten(3) + return xq_out.type_as(xq), xk_out.type_as(xk) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +# Adapted from transformers.models.llama.modeling_llama.eager_attention_forward -> llama4 doesn't cast attn weights to fp32 +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs, +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +# Adapted from transformers.models.llama.modeling_llama.eager_attention_forward -> llama4 doesn't cast attn weights to fp32 +def vision_eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs, +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * module.head_dim**-0.5 + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class Llama4TextAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Llama4TextConfig, layer_idx): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_attention_heads = config.num_attention_heads + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.num_key_value_heads = config.num_key_value_heads + self.scaling = self.head_dim**-0.5 + self.attn_scale = config.attn_scale + self.floor_scale = config.floor_scale + self.attn_temperature_tuning = config.attn_temperature_tuning + self.attention_dropout = config.attention_dropout + self.is_causal = True + self.use_rope = config.no_rope_layers[layer_idx] + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + if self.config.use_qk_norm and self.use_rope: + self.qk_norm = Llama4TextL2Norm(config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: torch.Tensor | None, + past_key_values: Cache | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape) + key_states = self.k_proj(hidden_states).view(*input_shape, -1, self.head_dim) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + if self.use_rope: # the 16E model skips rope for long context on certain layers + query_states, key_states = apply_rotary_emb( + query_states, key_states, position_embeddings.to(query_states.device) + ) + + if hasattr(self, "qk_norm"): # the 128E model does not use qk_norm + query_states = self.qk_norm(query_states) + key_states = self.qk_norm(key_states) + + # Use temperature tuning from https://huggingface.co/papers/2501.19399) to NoROPE layers + if self.attn_temperature_tuning and not self.use_rope: + past_seen_tokens = past_key_values.get_seq_length(self.layer_idx) if past_key_values is not None else 0 + positions = torch.arange(hidden_states.shape[1], device=hidden_states.device) + past_seen_tokens + attn_scales = ( + torch.log1p(torch.floor((positions.float() + 1.0) / self.floor_scale)) * self.attn_scale + 1.0 + ) + attn_scales = attn_scales.view((1, input_shape[-1], 1, 1)).expand((*input_shape, 1, 1)) # batch size > 1 + query_states = (query_states * attn_scales).to(query_states.dtype) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Llama4TextDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config, layer_idx): + super().__init__() + self.hidden_size = config.hidden_size + self.layer_idx = layer_idx + self.self_attn = Llama4TextAttention(config, layer_idx) + self.is_moe_layer = layer_idx in config.moe_layers + if self.is_moe_layer: # the 128E model interleaves dense / sparse + self.feed_forward = Llama4TextMoe(config) + else: + self.feed_forward = Llama4TextMLP(config, intermediate_size=config.intermediate_size_mlp) + + self.input_layernorm = Llama4TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Llama4TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + attention_states, _ = self.self_attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + hidden_states = residual + attention_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.feed_forward(hidden_states) + if self.is_moe_layer: + hidden_states, _ = hidden_states + hidden_states = residual + hidden_states.view(residual.shape) + return hidden_states + + +@auto_docstring +class Llama4PreTrainedModel(PreTrainedModel): + config: Llama4Config + input_modalities = ("image", "text") + supports_gradient_checkpointing = True + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = False + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + + @torch.no_grad() + def _init_weights(self, module): + super()._init_weights(module) + std = ( + self.config.initializer_range + if hasattr(self.config, "initializer_range") + else self.config.text_config.initializer_range + ) + if isinstance(module, Llama4TextExperts): + init.normal_(module.gate_up_proj, mean=0.0, std=std) + init.normal_(module.down_proj, mean=0.0, std=std) + elif isinstance(module, Llama4VisionRotaryEmbedding): + init.copy_(module.freqs_ci, module._compute_freqs_ci(module.config)) + elif isinstance(module, Llama4VisionModel): + init.normal_(module.class_embedding, std=module.scale) + init.normal_(module.positional_embedding_vlm, std=module.scale) + + +@auto_docstring +class Llama4TextModel(Llama4PreTrainedModel): + _no_split_modules = ["Llama4TextDecoderLayer"] + base_model_prefix = "model" + input_modalities = ("text",) + config: Llama4TextConfig + _can_record_outputs = { + "attentions": Llama4TextAttention, + "hidden_states": Llama4TextDecoderLayer, + "router_logits": Llama4TextMoe, + } + + def __init__(self, config: Llama4TextConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Llama4TextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = Llama4TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Llama4TextRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids.to(self.embed_tokens.weight.device)) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + # It may already have been prepared by e.g. `generate` + if not isinstance(causal_mask_mapping := attention_mask, dict): + # Prepare mask arguments + mask_kwargs = { + "config": self.config, + "inputs_embeds": inputs_embeds, + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "position_ids": position_ids, + } + # Create the masks + causal_mask_mapping = { + "full_attention": create_causal_mask(**mask_kwargs), + "chunked_attention": create_chunked_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + freq_cis = self.rotary_emb(hidden_states, position_ids) + + for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[self.config.layer_types[i]], + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=freq_cis, + **kwargs, + ) + hidden_states = self.norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + ) + + +class Llama4ForCausalLM(Llama4PreTrainedModel, GenerationMixin): + _no_split_modules = ["Llama4TextDecoderLayer"] + base_model_prefix = "language_model" + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + config: Llama4TextConfig + + def __init__(self, config: Llama4TextConfig): + super().__init__(config) + self.model = Llama4TextModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | CausalLMOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Example: + + ```python + >>> from transformers import AutoTokenizer, Llama4ForCausalLM + + >>> model = Llama4ForCausalLM.from_pretrained("meta-llama4/Llama4-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama4/Llama4-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs[0] + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@auto_docstring( + custom_intro=""" + Base class for Llava causal language model (or autoregressive) outputs. + """ +) +@dataclass +class Llama4CausalLMOutputWithPast(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + image_hidden_states (`torch.FloatTensor`, *optional*): + A `torch.FloatTensor` of size (batch_size, num_images, sequence_length, hidden_size)`. + image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. + """ + + loss: torch.FloatTensor | None = None + logits: torch.FloatTensor | None = None + past_key_values: Cache | None = None + hidden_states: tuple[torch.FloatTensor] | None = None + attentions: tuple[torch.FloatTensor] | None = None + image_hidden_states: torch.FloatTensor | None = None + + +class Llama4VisionMLP2(torch.nn.Module): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.fc1 = nn.Linear(self.intermediate_size, config.projector_input_dim, bias=False) + self.fc2 = nn.Linear(config.projector_output_dim, config.projector_output_dim, bias=False) + self.activation_fn = nn.GELU() # ACT2FN[config.hidden_act] + self.dropout = config.projector_dropout + + def forward(self, hidden_states): + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = F.dropout(hidden_states, p=self.dropout, training=self.training) + return self.activation_fn(self.fc2(hidden_states)) + + +class Llama4MultiModalProjector(nn.Module): + def __init__(self, config): + super().__init__() + self.linear_1 = nn.Linear( + config.vision_config.vision_output_dim, + config.text_config.hidden_size, + bias=False, + ) + + def forward(self, image_features): + hidden_states = self.linear_1(image_features) + return hidden_states + + +def pixel_shuffle(input_tensor, shuffle_ratio): + # input_tensor: [batch_size, num_patches, channels] + batch_size, num_patches, channels = input_tensor.shape + patch_size = int(math.sqrt(num_patches)) + + input_tensor = input_tensor.view(batch_size, patch_size, patch_size, -1) + batch_size, height, width, channels = input_tensor.size() + + reshaped_tensor = input_tensor.view(batch_size, height, int(width * shuffle_ratio), int(channels / shuffle_ratio)) + reshaped_tensor = reshaped_tensor.permute(0, 2, 1, 3).contiguous() + + reshaped_tensor = reshaped_tensor.view( + batch_size, int(height * shuffle_ratio), int(width * shuffle_ratio), int(channels / (shuffle_ratio**2)) + ) + reshaped_tensor = reshaped_tensor.permute(0, 2, 1, 3).contiguous() + + output_tensor = reshaped_tensor.view(batch_size, -1, reshaped_tensor.shape[-1]) + return output_tensor + + +class Llama4VisionPixelShuffleMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.pixel_shuffle_ratio = config.pixel_shuffle_ratio + self.inner_dim = int(config.projector_input_dim // (self.pixel_shuffle_ratio**2)) + self.output_dim = config.projector_output_dim + self.mlp = Llama4VisionMLP2(config) + + def forward(self, encoded_patches: torch.Tensor) -> torch.Tensor: + encoded_patches = pixel_shuffle(encoded_patches, self.pixel_shuffle_ratio) + return self.mlp(encoded_patches) + + +# TODO there is a different RoPE for vision encoder, defined as below +def reshape_for_broadcast(freqs_ci: torch.Tensor, query: torch.Tensor): + ndim = query.ndim + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(query.shape)] + return freqs_ci.view(*shape) + + +def vision_apply_rotary_emb( + query: torch.Tensor, + key: torch.Tensor, + freqs_ci: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + query_ = torch.view_as_complex(query.float().reshape(*query.shape[:-1], -1, 2)) + key_ = torch.view_as_complex(key.float().reshape(*key.shape[:-1], -1, 2)) + freqs_ci = reshape_for_broadcast(freqs_ci=freqs_ci, query=query_) # freqs_ci[:,:,None,:] + freqs_ci = freqs_ci.to(query_.device) + query_out = torch.view_as_real(query_ * freqs_ci).flatten(3) + key_out = torch.view_as_real(key_ * freqs_ci).flatten(3) + return query_out.type_as(query), key_out.type_as(key) # but this drops to 8e-3 + + +class Llama4VisionAttention(nn.Module): + def __init__(self, config: Llama4VisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = config.hidden_size // config.num_attention_heads + self.num_key_value_groups = 1 + self.attention_dropout = config.attention_dropout + self.scaling = self.head_dim**-0.5 + + self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.embed_dim, bias=True) + + def forward( + self, + hidden_states: torch.Tensor, + freqs_ci: torch.Tensor, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape) + key_states = self.k_proj(hidden_states).view(hidden_shape) + value_states = self.v_proj(hidden_states).view(hidden_shape) + + query_states, key_states = vision_apply_rotary_emb(query_states, key_states, freqs_ci=freqs_ci) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, vision_eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + None, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=None, # TODO Might be enforced here for TP compatibility as scaling is not just sqrt(head_dim) + is_causal=False, # HAS TO BE ENFORCED + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class Llama4VisionMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = nn.GELU() # ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=True) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=True) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +class Llama4VisionEncoderLayer(GradientCheckpointingLayer): + def __init__(self, config: Llama4VisionConfig): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = Llama4VisionAttention(config) + self.mlp = Llama4VisionMLP(config) + + self.input_layernorm = nn.LayerNorm(config.hidden_size) + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size) + + def forward( + self, + hidden_state: torch.Tensor, + freqs_ci: torch.Tensor, + attention_mask: torch.Tensor | None = None, + output_attentions: bool | None = None, + ): + # Self Attention + residual = hidden_state + + hidden_state = self.input_layernorm(hidden_state) + + hidden_state, attn_weights = self.self_attn( + hidden_state, + freqs_ci=freqs_ci, + attention_mask=attention_mask, + ) + hidden_state = residual + hidden_state + + # Feed forward + residual = hidden_state + hidden_state = self.post_attention_layernorm(hidden_state) + hidden_state = self.mlp(hidden_state) + hidden_state = residual + hidden_state + + outputs = (hidden_state,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +class Llama4VisionEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`Llama4VisionEncoderLayer`]. + + Args: + config: Llama4VisionConfig + """ + + def __init__(self, config: Llama4VisionConfig): + super().__init__() + self.config = config + self.layers = nn.ModuleList([Llama4VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + self.config = config + + def forward( + self, + hidden_states: torch.Tensor, + freqs_ci: torch.Tensor, # TODO move this to an attribute instead of keeping it around + attention_mask: torch.Tensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + ) -> tuple | BaseModelOutput: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + for encoder_layer in self.layers: + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + layer_outputs = encoder_layer( + hidden_state=hidden_states, + attention_mask=attention_mask, + output_attentions=output_attentions, + freqs_ci=freqs_ci, + ) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = layer_outputs[0] + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +class Llama4UnfoldConvolution(nn.Module): + def __init__(self, config): + super().__init__() + kernel_size = config.patch_size + if isinstance(kernel_size, int): + kernel_size = (kernel_size, kernel_size) + self.unfold = torch.nn.Unfold(kernel_size=kernel_size, stride=config.patch_size) + self.linear = nn.Linear( + config.num_channels * kernel_size[0] * kernel_size[1], + config.hidden_size, + bias=False, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.unfold(hidden_states) + hidden_states = hidden_states.permute(0, 2, 1) + hidden_states = self.linear(hidden_states) + return hidden_states + + +class Llama4VisionRotaryEmbedding(nn.Module): + def __init__(self, config: Llama4VisionConfig): + super().__init__() + self.config = config + self.register_buffer("freqs_ci", self._compute_freqs_ci(config), persistent=False) + + @staticmethod + def _compute_freqs_ci(config): + idx = config.image_size // config.patch_size + img_idx = torch.arange(idx**2, dtype=torch.int32).reshape(idx**2, 1) + img_idx = torch.cat([img_idx, img_idx[:1]], dim=0) + img_idx[-1, -1] = -2 # ID_CLS_TOKEN + frequencies_x = img_idx % idx # get the coordinates of the 2d matrix along x + frequencies_y = img_idx // idx # get the coordinates of the 2d matrix along y + freq_dim = config.hidden_size // config.num_attention_heads // 2 + rope_freq = 1.0 / ( + config.rope_parameters["rope_theta"] + ** (torch.arange(0, freq_dim, 2)[: (freq_dim // 2)].float() / freq_dim) + ) + freqs_x = ((frequencies_x + 1)[..., None] * rope_freq[None, None, :]).repeat_interleave(2, dim=-1) + freqs_y = ((frequencies_y + 1)[..., None] * rope_freq[None, None, :]).repeat_interleave(2, dim=-1) + freqs = torch.cat([freqs_x, freqs_y], dim=-1).float().contiguous()[..., ::2] + freqs = freqs.masked_fill(img_idx.reshape(-1, 1, 1) < 0, 0) + freq_cis = torch.view_as_complex(torch.stack([torch.cos(freqs), torch.sin(freqs)], dim=-1)) + return freq_cis # idx**2, idx**2, idx * 2 + + def forward(self, hidden_states): + return self.freqs_ci.to(hidden_states.device) + + +class Llama4VisionModel(Llama4PreTrainedModel): + base_model_prefix = "vision_model" + input_modalities = ("image",) + _no_split_modules = ["Llama4VisionEncoderLayer"] + config: Llama4VisionConfig + + def __init__(self, config: Llama4VisionConfig): + super().__init__(config) + self.image_size = config.image_size + self.patch_size = config.patch_size + self.hidden_size = config.hidden_size + self.num_channels = config.num_channels + + self.num_patches = (self.image_size // self.patch_size) ** 2 + 1 + self.scale = config.hidden_size**-0.5 + + self.patch_embedding = Llama4UnfoldConvolution(config) + + self.class_embedding = nn.Parameter(self.scale * torch.randn(self.hidden_size)) + self.positional_embedding_vlm = nn.Parameter(self.scale * torch.randn(self.num_patches, self.hidden_size)) + self.rotary_embedding = Llama4VisionRotaryEmbedding(config) + + # layer norms + self.layernorm_pre = nn.LayerNorm(self.hidden_size) + self.layernorm_post = nn.LayerNorm(self.hidden_size) + + # encoders + self.model = Llama4VisionEncoder(config) + self.vision_adapter = Llama4VisionPixelShuffleMLP(config) + self.post_init() + + def get_input_embeddings(self): + """ + This function is used to fetch the first embedding layer to activate grads on inputs. + """ + return self.patch_embedding + + def forward( + self, + pixel_values: torch.Tensor, + attention_mask: torch.Tensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> BaseModelOutputWithPooling | tuple[torch.Tensor, ...]: + r""" + + Example: + + ```python + >>> from PIL import Image + >>> import httpx + >>> from io import BytesIO + >>> from transformers import AutoProcessor, MllamaVisionModel + + >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" + >>> model = MllamaVisionModel.from_pretrained(checkpoint) + >>> processor = AutoProcessor.from_pretrained(checkpoint) + + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + >>> inputs = processor(images=image, return_tensors="pt") + + >>> output = model(**inputs) + + >>> print(output.last_hidden_state.shape) + torch.Size([1, 1, 4, 1025, 7680]) + ``` + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + # num_concurrent_media and num_chunks are both currently 1 + batch_size_times_num_tiles, num_channels, height, width = pixel_values.shape + num_concurrent_media = 1 + num_chunks = 1 + hidden_state = self.patch_embedding(pixel_values) + _, num_patches, hidden_dim = hidden_state.shape + + # Add cls token + hidden_state = hidden_state.reshape( + batch_size_times_num_tiles * num_concurrent_media * num_chunks, num_patches, hidden_dim + ) + class_embedding = self.class_embedding.expand(hidden_state.shape[0], 1, hidden_state.shape[-1]) + hidden_state = torch.cat([hidden_state, class_embedding], dim=1) + num_patches += 1 + + # Position embeddings + hidden_state = hidden_state.reshape( + batch_size_times_num_tiles * num_concurrent_media, num_chunks, num_patches, hidden_dim + ) + positional_embedding = self.positional_embedding_vlm.to(dtype=hidden_state.dtype, device=hidden_state.device) + hidden_state = hidden_state + positional_embedding + + hidden_state = self.layernorm_pre(hidden_state) + + hidden_state = hidden_state.view(batch_size_times_num_tiles, -1, hidden_dim) + freqs_ci = self.rotary_embedding(pixel_values) + + output = self.model( + hidden_state, + attention_mask=None, + output_hidden_states=output_hidden_states, + output_attentions=output_attentions, + freqs_ci=freqs_ci, + ) + + hidden_state = output.last_hidden_state + + hidden_state = self.layernorm_post(hidden_state) + + hidden_state = hidden_state[:, :-1, :] + + # now, we use Llama4VisionPixelShuffle + mlp to project embeddings + hidden_state = self.vision_adapter(hidden_state) + + hidden_states = output.hidden_states if output_hidden_states else None + + if output_attentions: + attentions = output[2] + else: + attentions = None + + if not return_dict: + return tuple(v for v in [hidden_state, hidden_states, attentions] if v is not None) + + return BaseModelOutputWithPooling( + last_hidden_state=hidden_state, + hidden_states=hidden_states, + attentions=attentions, + ) + + +class Llama4ForConditionalGeneration(Llama4PreTrainedModel, GenerationMixin): + _no_split_modules = ["Llama4TextDecoderLayer", "Llama4VisionEncoderLayer"] + _tp_plan = {} + base_model_prefix = "model" + config: Llama4Config + + def __init__(self, config: Llama4Config): + super().__init__(config) + self.vision_model = Llama4VisionModel(config.vision_config) + + self.multi_modal_projector = Llama4MultiModalProjector(config) + self.language_model = Llama4ForCausalLM(config.text_config) + self.vocab_size = config.text_config.vocab_size + if hasattr(self.config, "pad_token_id"): + self.pad_token_id = self.config.pad_token_id + else: + self.pad_token_id = self.config.text_config.pad_token_id or -1 + + self.post_init() + + def get_output_embeddings(self): + return self.language_model.get_output_embeddings() + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def set_decoder(self, decoder): + self.language_model.set_decoder(decoder) + + def get_decoder(self): + return self.language_model.get_decoder() + + @merge_with_config_defaults + @capture_outputs(tie_last_hidden_states=False) + @auto_docstring(custom_intro="Obtains image last hidden states from the vision tower and apply al projection.") + def get_image_features( + self, + pixel_values: torch.FloatTensor, + vision_feature_select_strategy: str, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPooling: + r""" + pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`) + The tensors corresponding to the input images. + vision_feature_select_strategy (`str`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"` + """ + kwargs = {k: v for k, v in kwargs.items() if v is not None} + return self.vision_model(pixel_values, **kwargs) + + def get_placeholder_mask( + self, input_ids: torch.LongTensor, inputs_embeds: torch.FloatTensor, image_features: torch.FloatTensor + ): + """ + Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is + equal to the length of multimodal features. If the lengths are different, an error is raised. + """ + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device) + ) + special_image_mask = special_image_mask.all(-1) + else: + special_image_mask = input_ids == self.config.image_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + torch_compilable_check( + inputs_embeds[special_image_mask].numel() == image_features.numel(), + f"Image features and image tokens do not match, tokens: {n_image_tokens}, features: {image_features.shape[0]}", + ) + return special_image_mask + + @merge_with_config_defaults + @capture_outputs(tie_last_hidden_states=False) + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + pixel_values: torch.FloatTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + vision_feature_select_strategy: str | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | Llama4CausalLMOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Example: + + ```python + >>> from PIL import Image + >>> import httpx + >>> from io import BytesIO + >>> from transformers import AutoProcessor, LlavaForConditionalGeneration + + >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf") + >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf") + + >>> prompt = "USER: \nWhat's the content of the image? ASSISTANT:" + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + + >>> inputs = processor(images=image, text=prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(**inputs, max_new_tokens=15) + >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "USER: \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed" + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if pixel_values is not None and inputs_embeds is not None: + raise ValueError( + "You cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one" + ) + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_features = self.get_image_features( + pixel_values=pixel_values, + vision_feature_select_strategy=vision_feature_select_strategy, + return_dict=True, + ).last_hidden_state + + vision_flat = image_features.view(-1, image_features.size(-1)) + projected_vision_flat = self.multi_modal_projector(vision_flat).to( + inputs_embeds.device, inputs_embeds.dtype + ) + special_image_mask = self.get_placeholder_mask( + input_ids, inputs_embeds=inputs_embeds, image_features=projected_vision_flat + ) + inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, projected_vision_flat) + + outputs = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + logits_to_keep=logits_to_keep, + **kwargs, + ) + + logits = outputs[0] + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + if attention_mask is not None: + # we use the input attention mask to shift the logits and labels, because it is 2D. + # we also crop attn mask in case it is longer, which happens in PrefixTuning with peft + shift_attention_mask = attention_mask[:, -(logits.shape[1] - 1) :].to(logits.device) + shift_logits = logits[..., :-1, :][shift_attention_mask.to(logits.device) != 0].contiguous() + shift_labels = labels[..., 1:][shift_attention_mask.to(labels.device) != 0].contiguous() + else: + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1).to(shift_logits.device) + ) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return Llama4CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_hidden_states=image_features if pixel_values is not None else None, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + inputs_embeds=None, + pixel_values=None, + attention_mask=None, + logits_to_keep=None, + is_first_iteration=False, + **kwargs, + ): + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model + + model_inputs = self.language_model.prepare_inputs_for_generation( + input_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + logits_to_keep=logits_to_keep, + is_first_iteration=is_first_iteration, + **kwargs, + ) + + if is_first_iteration or not kwargs.get("use_cache", True): + # Pixel values are used only in the first iteration if available + # In subsequent iterations, they are already merged with text and cached + # NOTE: first iteration doesn't have to be prefill, it can be the first + # iteration with a question and cached system prompt (continue generate from cache) + model_inputs["pixel_values"] = pixel_values + + return model_inputs + + +__all__ = [ + "Llama4PreTrainedModel", + "Llama4TextModel", + "Llama4VisionModel", + "Llama4ForCausalLM", + "Llama4ForConditionalGeneration", +] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_online_rejected.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_online_rejected.txt new file mode 100644 index 0000000000000000000000000000000000000000..00ae5bd9a04506ffa42c0d2e2349a108358c2684 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_online_rejected.txt @@ -0,0 +1 @@ +rank=0 worker=0 doc=10 ntok=3743 reason=bigram_repeat>=16(count=44,text=medical marijuana) text=If California is the cradle of the medical marijuana movement, then Venice Beach might be where it reached maturity — and some would say, where it ran wild.\n\nKnown for its eclectic mix of tattoo parlors, street performers and surf shops, the bohemian Los Angeles neighborhood has long been home to the city’s younger creative and artistic crowds. But the boardwalk culture took on a new vibe after California became the first state to pass a medical marijuana law in 1996. Today, the strip is home to several pipe stores and marijuana dispensaries like The Venice Beach Care Center, Nile Collective and Green Cure Venice, where “budtenders” and “budistas” offer menus of buds for clientele carrying a California doctor’s recommendation for medical marijuana. Most sell a smorgasbord of marijuana-infused edible products like brownies and cookies, but also juices, olive oils and beef jerky infused with THC, marijuana’s mind-altering ingredient.\n\nDrexel sociologist and School of Public Health Associate Professor Stephen Lankenau was living and working in Los Angeles in 2007 when the rapid growth in dispensaries — from four in 2005 to 186 two years later — motivated the city to impose a moratorium while it developed regulations. Entrepreneurs quickly took advantage of loopholes in the moratorium, which was never enforced, and over the next several years, the number of dispensaries in the city exploded — up to 1,000 by some estimates.\n\n“They became as common as Starbucks,” Lankenau says.\n\nEditor’s Note: In May, residents passed a bill calling for the closure of all dispensaries that had opened since the 2007 moratorium, which is intended to bring down the number of dispensaries to less than 140.\n\nThe number of medical conditions that qualify for marijuana treatment also exploded. Though the initial law focused on chronic health conditions like cancer and glaucoma, later changes permitted psychological conditions like insomnia and anxiety. According to one physician’s survey, California doctors have recommended medical marijuana for more than 215 conditions.\n\nLankenau knew the large number of medical marijuana dispensaries in the city and its diverse population of young adults made Los Angeles fertile ground for a scientific study.\n\n“We started to see a lot of young adults getting recommendations for these conditions,” Lankenau recalls. “That should raise some red flags as to whether there might be some primarily recreational users mixed in with legitimate medical users. I wanted to look into that.”\n\nAn unstudied area\n\nLankenau recently got his wish.\n\nThe National Institute on Drug Abuse, part of the National Institutes of Health, has granted Lankenau $3.3 million over five years to conduct a study of medical marijuana use among young adults. Working with co-investigator Ellen Iverson at Children’s Hospital Los Angeles, Lankenau’s team will enroll 380 marijuana users this fall and expects to complete baseline interviews and surveys by early 2014. Half of the cohort will be qualified medical marijuana patients and half will be recreational users. Participants will complete annual follow-up interviews for three years.\n\nIt will be the country’s first large-scale federally funded case study of medical marijuana use among 18- to 26-year-olds.\n\nNIH research grants — particularly the large and lucrative R01 grant type awarded to Lankenau — to study medical marijuana aren’t easy to come by. Possession and distribution of marijuana is still illegal under federal law, and despite supportive statements from President Obama, under his administration the Drug Enforcement Agency has stepped up raids of medical marijuana dispensaries in states where licensing guidelines are unclear.\n\nA search of medical marijuana studies in NIH Reporter, the agency’s database of the research it funds, turns up just 15 active studies nationwide, and only about half of those deal with population behavior similar to Lankenau’s.\n\n“It’s a controversial topic,” Lankenau says.\n\nLankenau’s interest in the relationship between medical marijuana and young people stems from his dissertation research on homeless panhandlers in Washington, D.C., where he saw firsthand how drug and alcohol problems affected the health of a vulnerable population. Later, as a postdoctoral fellow, he focused his work on substance use and HIV/AIDS.\n\n“It became clear to me that substance use was at the heart of a whole range of social and public health problems — sometimes the cause and sometimes the effect — such as poverty, violence, disease, social justice, psychological and physical well-being,” he says. “Substance use is a very dynamic field for research.”\n\n“What we’re really trying to find out is, is this a good policy for young adults? Does it seem to result in improved health overall, or does it have unintended consequences that may give pause to other states?”\n\n– Stephen Lankenau\n\nMost of the studies done on medical marijuana to date have focused solely on quantifying whether illegal marijuana use goes up or down in states after they pass medical marijuana legislation. Those studies have typically been smaller in scale, or involve secondary analysis of survey data without direct recruitment of medical marijuana-using subjects.\n\nThe federal stance on funding medical marijuana research has softened as the number of states that permit and regulate medical marijuana sales and use has continued to grow. So far, 20 states plus the District of Columbia have passed medical marijuana laws, and this past November, voters in Colorado and Washington (where marijuana for medical use was already legal) passed initiatives that legalized marijuana for recreational use. Bills that would allow medical marijuana are pending in four other states, including Pennsylvania.\n\nSee an Interactive Map Showing Medical Marijuana Legislation\n\nLighting Up by Law\n\nCall it reefer madness, or call it the arrival of a sane public health policy at last—depending on your viewpoint. In 2013, lawmakers filed bills in 30 states and Washington, D.C., favorable to marijuana possession for personal or medical use. Two states, Illinois and New Hampshire, effectively legalized medical marijuana this year, bringing the total to 20 states and the nation’s capital where medical marijuana is legal. In addition, both Colorado and Washington passed laws legalizing recreational marijuana in last November’s election. Legislatures in four additional states, including Pennsylvania, are considering bills that would legalize medical marijuana.\n\nThe timing is ripe for more research.\n\nStarting this fall, Lankenau’s team will recruit study participants by “hanging out” in areas like Venice Beach that have a large number of dispensaries. Participants receive $25 as an incentive to give the initial interview.\n\n“It takes a little finesse and experience to do it in a way that makes the person comfortable talking about their drug use,” Lankenau says.\n\nLankenau’s collaborator, Ellen Iverson, says the age group targeted by the study is important because marijuana use (medical or illicit) typically peaks between ages 19 to 21.\n\n“These often are the most vulnerable times of their lives — years when developmentally they are making significant, culturally adult-oriented decisions about their lives and their identities are very much in flux,” Iverson says. “There really hasn’t been a body of strong, rigorous evidence that looks at how young people make decisions about marijuana use. Here’s an opportunity to learn what kinds of issues they are using marijuana for and, especially as policy is being formulated and evolving, to understand how it affects [them].”\n\nA better understanding of why young people turn to the drug could help better inform policies on medical marijuana, she says.\n\n“What we’re really trying to find out is, is this a good policy for young adults?” Lankenau says. “Does it seem to result in improved health overall, or does it have unintended consequences that may give pause to other states? Or should it be scaled back somehow?”\n\nSmoking out the facts\n\nOne hypothesis that Lankenau and his team want to test is the so-called “gateway theory” that marijuana users go on to use harder, more addictive drugs.\n\n“It’s true that most people who use hard drugs like cocaine and heroin have used marijuana, [but] most people who have smoked marijuana don’t go on to use harder drugs,” Lankenau says. “There’s not really a lot of data that validates or supports that theory, but it’s sort of well out there in the popular imagination that one of the risks of marijuana is it can lead to harder drug use.”\n\nIn fact, Lankenau suspects the reverse may be true.\n\nThe idea that medical marijuana may potentially offer a protective effect against other drug use grew out of another large-scale study funded by the National Institute on Drug Abuse that he conducted at Children’s Hospital Los Angeles. The study was focused on prescription drug abuse in young adults, but participants were asked if they used medical marijuana or smoked it recreationally. The team found generally lower rates of drug use overall with medical marijuana patients compared with recreational marijuana users.\n\nA separate study along the same lines by a colleague in the Netherlands showed hard drug users were more likely to “step off” using those substances once marijuana use was decriminalized.\n\n“It’s sort of turning the ‘gateway model’ on its head a little,”\n\nLankenau says.\n\nResearchers will also seek to gauge whether young medical marijuana patients experience an overall improvement in physical and psychological health. Study participants will be asked a series of questions about the basis for their marijuana treatment recommendation and their medical history to discern whether they are, in fact, getting the prescription for legitimate purposes. They’ll also be asked about previously diagnosed conditions and what type of experience participants had using other drugs prescribed for it.\n\n“Maybe they got a prescription for them, but those prescriptions didn’t seem to work the way they wanted or it didn’t result in the level of improved health that they desired,” Lankenau says. “Is [medical marijuana] effective in terms of improving physical or psychological health?”\n\nA third component of the study will look at the dispensaries to see how they’re set up, how they service patients and what, if any, impact they have on emerging adults in their communities.\n\n“Is there enough regulation? Are staff trained adequately to provide [health] information?” Lankenau says.\n\nThe findings on this issue could have the most impact for other states grappling with how to regulate the sale and distribution of medical marijuana. In Los Angeles, dispensaries became a lightning rod for controversy. When dispensaries started opening in neighborhoods across the city, residents blamed them for increased crime and noise. They pressured city officials to step up enforcement and shutter those that were operating illegally.\n\nSince 2008, police conducting raids of illegal marijuana businesses have made more than 74 arrests citywide. More than $2 million in cash has been seized, as well as assault weapons, nine kilograms of cocaine, and large amounts of other illegal drugs, according to Los Angeles City Council member Mitchell Englander, a reserve police officer who also chairs the council’s public safety committee. In his 12th Council district, he’s worked with city administrators to shutter more than 60 dispensaries he says were operating illegally.\n\n“I know firsthand about the crime and other negative impacts on our neighborhoods that the illegal storefront marijuana stores have had,” says Englander. “When California voters approved the Compassionate Use Act of 1996 legalizing medical marijuana, it is likely that they had clinics or pharmacies in mind, serving people with chronic debilitating illnesses, rather than the nearly 1,000 storefront marijuana stores that sprang up across Los Angeles.”\n\n“Based on what we find, we could have recommendations on whether dispensaries are a good model for distribution,” says Lankenau.\n\nA patchwork of policies\n\nAcross the country, thousands of licensed physicians have recommended medical marijuana to at least 600,000 patients in states with medical marijuana laws, and the rate of legalization has outpaced policymakers’ grasp of the long-term consequences of widespread availability of marijuana.\n\nThe emergence of marijuana into mainstream medical usage traces back to 1982 when the National Academy of Sciences’ Institute of Medicine confirmed the “therapeutic potential” of marijuana. More recently, 76 percent of 1,446 physicians polled in the New England Journal of Medicine said they would prescribe marijuana to treat pain from cancer. And in August, influential CNN chief medical correspondent Sanjay Gupta, MD, reversed his long-held position on the issue, concluding, “Sometimes marijuana is the only thing that works.”\n\nCalifornia became the first state to sanction medical marijuana use when voters adopted the Compassionate Use Act of 1996. Since then, the state and localities there and in other states have struggled to balance the desire of patients to have safe, easy access to the drug with a public safety interest in curbing potential exploitation of the system by recreational users or criminals. In the latest development, California’s Supreme Court held in May that dispensaries were legal but could be banned by local governments.\n\n“There really hasn’t been a body of strong, rigorous evidence that looks at how young people make decisions about marijuana use. Here’s an opportunity to learn what kinds of issues are they using marijuana for and, especially as policy is being formulated and evolving, to understand how it affects [them].”\n\n— Ellen Iverson\n\nMore than 3,000 miles away from Venice Beach in New Jersey, the issue isn’t whether there are too many dispensaries, but whether the state should loosen its restrictive medical marijuana law to make it easier for more dispensaries to open. New Jersey permitted the sale and use of a limited number of strains of medical marijuana three years ago, but so far only one dispensary has opened.\n\nOther states are trying to find the middle ground between the approaches taken in California and New Jersey. In Illinois, the most recent state to pass medical marijuana legislation, the bill’s chief architect and backer, Democratic Rep. Lou Lang, boasted at a bill signing ceremony that the law was “the most controlled and highly regulated bill in the country.” Illinois limits the number of cultivation centers to 22 and only allows for 60 dispensaries across the state, but it’s unclear how those numbers were selected.\n\nIn August, Attorney General Eric Holder announced that the Department of Justice would not seek to prosecute individual users and those who work in the industry in states that have legalized the drug for personal or medical use as long as states “implement strong and effective regulatory and enforcement systems” — a signal that the federal stance is softening further, but also a warning that the DOJ reserves the right to intervene if states don’t go far enough to protect public health and safety.\n\n“It’s an emerging trend around the country to pass this kind of law,” Lankenau says. “We hope the findings from our study may help inform other policies around the country as to whether this is a wise policy or whether it’s something that should be given further consideration.”\n\nA long, strange trip\n\nTimeline of recent medical marijuana laws in Los Angeles and California\n\n1992\n\n— Medical marijuana is approved in a few California cities including Santa Cruz.\n\n1996\n\n— California voters pass Proposition 215 (the Compassionate Use Act), which allows medical marijuana to be dispensed to patients with a doctor’s recommendation.\n\n2003\n\n— Senate Bill 420 (the Medical Marijuana Program Act) is written into law to clarify the scope and application of the Compassionate Use Act of 1996. It paves the way for a voluntary statewide medical marijuana identification card program.\n\n2005\n\n— Los Angeles has four known marijuana dispensaries, in Hancock Park, Van Nuys, Rancho Park and Cheviot Hills.\n\n2007\n\n— Concerned by a 2,350 percent increase in the number of medical marijuana dispensaries in Los Angeles during an approximately one-year period, the City Council establishes a moratorium on new medical marijuana storefronts on Aug. 1 and places restrictions on hours of operation and their proximity to schools, churches, parks and other dispensaries.\n\n2008\n\n— California Attorney General Jerry Brown issues a directive of guidelines for medical marijuana cooperatives including that they sell only to legitimate patients, operate as nonprofits, and only buy [marijuana] from fellow cooperative members.\n\n2009 —\n\nFederal officials announce that they will no longer try to block medical marijuana distribution and use in California.\n\nJanuary 2010 —\n\nThe Los Angeles City Council, without debate, approves an ordinance to shut down hundreds of medical marijuana dispensaries and impose strict rules on the operation and locations of cannabis clubs, shops and collectives. The ordinance caps the number of dispensaries at 70 (with the exception of those registered with the city in 2007).\n\n2012 —\n\nThe Los Angeles City Council banned all marijuana dispensaries, only to repeal the ban amid opposition.\n\nMay 6, 2013 — The California Supreme Court rules that local governments can ban marijuana dispensaries, the Los Angeles Times reports.\n\nMay 21, 2013\n\n— By a margin of nearly 63 percent, Los Angeles voters pass Proposition D, which aims to cap medical marijuana dispensaries to 135, forcing more than 1,000 shops that don’t meet the new legal qualifications to close.\n\nSource: Timeline reprinted with permission from Shana Ting Lipton, About.com (exception: the May 6 entry was sourced from the Los Angeles Times). diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_eval_hzj_c1tov_power2_decode_gamma_finesweep_16_32_gamma2_append_20260606_162525.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_eval_hzj_c1tov_power2_decode_gamma_finesweep_16_32_gamma2_append_20260606_162525.log new file mode 100644 index 0000000000000000000000000000000000000000..092d0937c5e8cf0e8cc942a6520fba7d4cc25985 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_eval_hzj_c1tov_power2_decode_gamma_finesweep_16_32_gamma2_append_20260606_162525.log @@ -0,0 +1,1639 @@ +[2026-06-06T16:28:25+00:00] ckpt=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +[2026-06-06T16:28:25+00:00] vocab=32100 gpu=3 n=64 chunk_n=8 +[2026-06-06T16:28:25+00:00] steps=16 32 +[2026-06-06T16:28:25+00:00] m=-1.1 -1.0 -0.9 -0.8 -0.7 -0.6 +[2026-06-06T16:28:25+00:00] s=0.5 0.6 0.7 0.8 +[2026-06-06T16:28:25+00:00] gamma=2.0 +[2026-06-06T16:28:25+00:00] run steps=16 schedule=uniform gamma=0.5 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:28:25+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=0 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=1 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=2 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=3 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=4 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=5 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=6 +[2026-06-06T16:28:25+00:00] skip existing decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 149.46344105270597 5.567153841746121 0.10837445973699537 0.6552226224231742 0.061536339392453177 63 66 64059 65246 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 283.2206652052596 5.694026544379917 0.11525091799265606 0.6971309201292555 0.02689514483884129 0 0 56119 61275 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:28:38+00:00] done +[2026-06-06T16:28:38+00:00] run steps=16 schedule=uniform gamma=1.0 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:28:38+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=0 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=1 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=2 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=3 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=4 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=5 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=6 +[2026-06-06T16:28:38+00:00] skip existing decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 149.67404725591572 5.574005557055492 0.10648887796485192 0.6587602734465012 0.058851542337470815 63 64 63698 65096 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 281.0028438239289 5.698437146811535 0.11321556387168467 0.700320303307622 0.021930612978608665 0 0 55903 61193 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:28:52+00:00] done +[2026-06-06T16:28:52+00:00] run steps=16 schedule=uniform gamma=1.5 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:28:52+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=0 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=1 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=2 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=3 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=4 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=5 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=6 +[2026-06-06T16:28:52+00:00] skip existing decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 139.86550404246822 5.54337872458112 0.10146819424394139 0.6505306160060204 0.05588659888810394 64 64 63786 65114 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 251.78280870321518 5.657209802422787 0.1075795815354555 0.68969616048458 0.019474069852641863 0 0 56353 61415 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:29:05+00:00] done +[2026-06-06T16:29:05+00:00] run steps=16 schedule=uniform gamma=2.0 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:29:05+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=0 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=1 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=2 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=3 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=4 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=5 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=6 +[2026-06-06T16:29:05+00:00] skip existing decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 138.963496602318 5.510404176756993 0.09783476658476659 0.641886392604309 0.04978501228501229 63 63 63294 65120 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 230.5072297940538 5.599628328889021 0.10295733678086619 0.6754795649574169 0.017873303167420813 0 0 56770 61880 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:29:18+00:00] done +[2026-06-06T16:29:18+00:00] run steps=32 schedule=uniform gamma=0.5 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:29:18+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=0 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=1 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=2 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=3 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=4 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=5 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=6 +[2026-06-06T16:29:18+00:00] skip existing decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 44.45341392625776 5.1910241422474215 0.08713153142122501 0.5264245373984113 0.08376442499005173 63 63 65544 65338 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 88.09069964135364 5.369408698747378 0.09510524557300368 0.5745836048046242 0.03753758770464417 0 0 54560 59860 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:29:31+00:00] done +[2026-06-06T16:29:31+00:00] run steps=32 schedule=uniform gamma=1.0 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:29:31+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=0 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=1 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=2 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=3 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=4 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=5 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=6 +[2026-06-06T16:29:31+00:00] skip existing decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 42.541972618531155 5.246427767644686 0.08249881511153241 0.5269852918692475 0.07844726099652942 64 64 65827 65407 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 79.56673655927631 5.41458726920399 0.08960007971832858 0.5724037136071481 0.03285059456586727 0 0 55408 60212 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:29:44+00:00] done +[2026-06-06T16:29:44+00:00] run steps=32 schedule=uniform gamma=1.5 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:29:45+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=0 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=1 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=2 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=3 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=4 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=5 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=6 +[2026-06-06T16:29:45+00:00] skip existing decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 43.24270640780984 5.26093876366149 0.07914175167077031 0.5292102526457454 0.07359035923473367 64 64 65452 65389 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 77.77531129198442 5.419289705900844 0.08550228876439774 0.5718039397144368 0.029134235618792657 0 0 55665 60513 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma1p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:29:58+00:00] done +[2026-06-06T16:29:58+00:00] run steps=32 schedule=uniform gamma=2.0 m=na s=na tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:29:58+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=0 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=1 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=2 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=3 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=4 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=5 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=6 +[2026-06-06T16:29:58+00:00] skip existing decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 43.13908306493343 5.260133181368608 0.07706784173322317 0.5266306593019875 0.07149851586645858 63 63 65275 65358 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 75.92654605973213 5.414707253645095 0.08307357379644748 0.5676540440691384 0.02556363696336978 0 0 55786 60633 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64/sc1p0 +sc1p0 raw_full nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos nan nan nan nan nan 0 0 0 0 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_uniform_step103000_dgamma2p0_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:30:11+00:00] done +[2026-06-06T16:30:11+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.1 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:30:11+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:30:11+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:30:17+00:00] done decode=16 chunk=0 +[2026-06-06T16:30:17+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:30:22+00:00] done decode=16 chunk=1 +[2026-06-06T16:30:22+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:30:28+00:00] done decode=16 chunk=2 +[2026-06-06T16:30:28+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:30:33+00:00] done decode=16 chunk=3 +[2026-06-06T16:30:33+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:30:39+00:00] done decode=16 chunk=4 +[2026-06-06T16:30:39+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:30:45+00:00] done decode=16 chunk=5 +[2026-06-06T16:30:45+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:30:51+00:00] done decode=16 chunk=6 +[2026-06-06T16:30:51+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:30:56+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 47.952924811066396 4.7885336794641225 0.05803274118357675 0.40676207291763467 0.06701565815354593 55 64 64522 64567 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 74.15537204507984 4.861993155906204 0.06151194287121399 0.4310339166694028 0.03483542641385537 0 0 57201 60915 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:31:09+00:00] done +[2026-06-06T16:31:09+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.1 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:31:09+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:31:09+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:31:15+00:00] done decode=16 chunk=0 +[2026-06-06T16:31:15+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:31:21+00:00] done decode=16 chunk=1 +[2026-06-06T16:31:21+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:31:26+00:00] done decode=16 chunk=2 +[2026-06-06T16:31:26+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:31:32+00:00] done decode=16 chunk=3 +[2026-06-06T16:31:32+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:31:38+00:00] done decode=16 chunk=4 +[2026-06-06T16:31:38+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:31:43+00:00] done decode=16 chunk=5 +[2026-06-06T16:31:43+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:31:49+00:00] done decode=16 chunk=6 +[2026-06-06T16:31:49+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:31:54+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 48.5095574617719 4.8158959793181575 0.05920664950531898 0.41798867953787705 0.06970505225940514 55 58 64897 64486 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 75.49247210905823 4.898205701932096 0.06281361564911242 0.4433713928465664 0.033594919632133985 0 0 57466 60783 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:32:07+00:00] done +[2026-06-06T16:32:07+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.1 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:32:07+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:32:07+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:32:13+00:00] done decode=16 chunk=0 +[2026-06-06T16:32:13+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:32:18+00:00] done decode=16 chunk=1 +[2026-06-06T16:32:18+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:32:24+00:00] done decode=16 chunk=2 +[2026-06-06T16:32:24+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:32:29+00:00] done decode=16 chunk=3 +[2026-06-06T16:32:29+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:32:35+00:00] done decode=16 chunk=4 +[2026-06-06T16:32:35+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:32:40+00:00] done decode=16 chunk=5 +[2026-06-06T16:32:40+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:32:46+00:00] done decode=16 chunk=6 +[2026-06-06T16:32:46+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:32:52+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 46.824065037738094 4.812370515576132 0.061333828835106304 0.4243419015174977 0.0766788993666868 52 53 65641 64581 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 75.2527873968173 4.908448272456839 0.06535764375876578 0.4521483730446835 0.033149080108901906 0 0 57668 60605 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:33:05+00:00] done +[2026-06-06T16:33:05+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.1 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:33:05+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:33:05+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:33:10+00:00] done decode=16 chunk=0 +[2026-06-06T16:33:10+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:33:16+00:00] done decode=16 chunk=1 +[2026-06-06T16:33:16+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:33:21+00:00] done decode=16 chunk=2 +[2026-06-06T16:33:21+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:33:27+00:00] done decode=16 chunk=3 +[2026-06-06T16:33:27+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:33:33+00:00] done decode=16 chunk=4 +[2026-06-06T16:33:33+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:33:38+00:00] done decode=16 chunk=5 +[2026-06-06T16:33:38+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:33:44+00:00] done decode=16 chunk=6 +[2026-06-06T16:33:44+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:33:50+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 47.095211421517384 4.818316603432693 0.06272737124015047 0.4295776828286581 0.0801120795083363 49 53 66027 64597 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 76.87060222066185 4.924729680846503 0.06696413815898199 0.45851030425225997 0.03437448355643695 0 0 57838 60510 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:34:03+00:00] done +[2026-06-06T16:34:03+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.0 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:34:03+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:34:03+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:34:08+00:00] done decode=16 chunk=0 +[2026-06-06T16:34:08+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:34:14+00:00] done decode=16 chunk=1 +[2026-06-06T16:34:14+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:34:19+00:00] done decode=16 chunk=2 +[2026-06-06T16:34:19+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:34:25+00:00] done decode=16 chunk=3 +[2026-06-06T16:34:25+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:34:30+00:00] done decode=16 chunk=4 +[2026-06-06T16:34:30+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:34:36+00:00] done decode=16 chunk=5 +[2026-06-06T16:34:36+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:34:41+00:00] done decode=16 chunk=6 +[2026-06-06T16:34:41+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:34:47+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 50.54577940310764 4.882462178896528 0.0626817647422808 0.43424655415126157 0.05525648165336303 57 60 63668 64644 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 72.64335405367562 4.937732293332493 0.06580806522339337 0.45582407587864615 0.03253049226121839 0 0 57504 61573 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:35:00+00:00] done +[2026-06-06T16:35:00+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.0 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:35:00+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:35:00+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:35:06+00:00] done decode=16 chunk=0 +[2026-06-06T16:35:06+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:35:11+00:00] done decode=16 chunk=1 +[2026-06-06T16:35:11+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:35:17+00:00] done decode=16 chunk=2 +[2026-06-06T16:35:17+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:35:22+00:00] done decode=16 chunk=3 +[2026-06-06T16:35:22+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:35:28+00:00] done decode=16 chunk=4 +[2026-06-06T16:35:28+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:35:33+00:00] done decode=16 chunk=5 +[2026-06-06T16:35:33+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:35:39+00:00] done decode=16 chunk=6 +[2026-06-06T16:35:39+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:35:44+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 48.81881611178905 4.898484340342371 0.0621748017839445 0.43392953929539296 0.06603072348860257 58 62 64490 64576 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 78.05127984412306 4.9883409679755975 0.06613952722181039 0.46150171323141803 0.03136479696894819 0 0 56732 60705 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:35:58+00:00] done +[2026-06-06T16:35:58+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.0 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:35:58+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:35:58+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:36:03+00:00] done decode=16 chunk=0 +[2026-06-06T16:36:03+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:36:09+00:00] done decode=16 chunk=1 +[2026-06-06T16:36:09+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:36:14+00:00] done decode=16 chunk=2 +[2026-06-06T16:36:14+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:36:20+00:00] done decode=16 chunk=3 +[2026-06-06T16:36:20+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:36:25+00:00] done decode=16 chunk=4 +[2026-06-06T16:36:25+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:36:31+00:00] done decode=16 chunk=5 +[2026-06-06T16:36:31+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:36:36+00:00] done decode=16 chunk=6 +[2026-06-06T16:36:36+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:36:42+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 50.148846303997885 4.910734053838735 0.0634974585567074 0.4463584958131199 0.06652556120320732 59 64 64773 64727 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 80.86944685540466 5.003530725410337 0.0676175904446967 0.4750584054489816 0.03153842357238044 0 0 56868 60783 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:36:55+00:00] done +[2026-06-06T16:36:55+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-1.0 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:36:55+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:36:55+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:37:01+00:00] done decode=16 chunk=0 +[2026-06-06T16:37:01+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:37:06+00:00] done decode=16 chunk=1 +[2026-06-06T16:37:06+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:37:12+00:00] done decode=16 chunk=2 +[2026-06-06T16:37:12+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:37:17+00:00] done decode=16 chunk=3 +[2026-06-06T16:37:17+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:37:23+00:00] done decode=16 chunk=4 +[2026-06-06T16:37:23+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:37:29+00:00] done decode=16 chunk=5 +[2026-06-06T16:37:29+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:37:34+00:00] done decode=16 chunk=6 +[2026-06-06T16:37:34+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:37:40+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 48.00175297755785 4.9022213873699 0.06543689020147511 0.44857125185551705 0.07582762512949763 58 59 65498 64673 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 83.79549142924664 5.020699472969632 0.07035977921127884 0.48225181638319453 0.031987763516658906 0 0 56418 60148 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:37:53+00:00] done +[2026-06-06T16:37:53+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.9 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:37:53+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:37:53+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:37:58+00:00] done decode=16 chunk=0 +[2026-06-06T16:37:58+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:38:04+00:00] done decode=16 chunk=1 +[2026-06-06T16:38:04+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:38:09+00:00] done decode=16 chunk=2 +[2026-06-06T16:38:09+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:38:15+00:00] done decode=16 chunk=3 +[2026-06-06T16:38:15+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:38:21+00:00] done decode=16 chunk=4 +[2026-06-06T16:38:21+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:38:26+00:00] done decode=16 chunk=5 +[2026-06-06T16:38:26+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:38:32+00:00] done decode=16 chunk=6 +[2026-06-06T16:38:32+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:38:37+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 53.80189439761171 4.955191802322993 0.06494027593444242 0.45017516243035943 0.04764035927034785 49 54 62696 64798 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 73.23249425109923 5.002815630606312 0.06761903231508412 0.46868070062670736 0.03009753981134804 0 0 57536 62231 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:38:50+00:00] done +[2026-06-06T16:38:50+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.9 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:38:50+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:38:50+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:38:56+00:00] done decode=16 chunk=0 +[2026-06-06T16:38:56+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:39:01+00:00] done decode=16 chunk=1 +[2026-06-06T16:39:01+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:39:07+00:00] done decode=16 chunk=2 +[2026-06-06T16:39:07+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:39:12+00:00] done decode=16 chunk=3 +[2026-06-06T16:39:12+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:39:18+00:00] done decode=16 chunk=4 +[2026-06-06T16:39:18+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:39:23+00:00] done decode=16 chunk=5 +[2026-06-06T16:39:23+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:39:29+00:00] done decode=16 chunk=6 +[2026-06-06T16:39:29+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:39:34+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 54.79388471560767 4.998209592026316 0.0675535777746875 0.46599091442875246 0.055269704414468705 59 64 63390 64719 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 81.27747495858739 5.064995450861096 0.07109752329533443 0.4903076823001366 0.029694436765160263 0 0 56917 61493 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:39:47+00:00] done +[2026-06-06T16:39:47+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.9 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:39:47+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:39:47+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:39:53+00:00] done decode=16 chunk=0 +[2026-06-06T16:39:53+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:39:58+00:00] done decode=16 chunk=1 +[2026-06-06T16:39:58+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:40:04+00:00] done decode=16 chunk=2 +[2026-06-06T16:40:04+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:40:09+00:00] done decode=16 chunk=3 +[2026-06-06T16:40:09+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:40:15+00:00] done decode=16 chunk=4 +[2026-06-06T16:40:15+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:40:21+00:00] done decode=16 chunk=5 +[2026-06-06T16:40:21+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:40:26+00:00] done decode=16 chunk=6 +[2026-06-06T16:40:26+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:40:32+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 54.00761315740999 4.987457563123889 0.0665388362275588 0.46752644275375765 0.06163695124402728 58 63 64049 64669 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 84.75412177505275 5.073488135194577 0.07048068416396343 0.49525697527729085 0.031291982043972606 0 0 56759 61038 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:40:44+00:00] done +[2026-06-06T16:40:44+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.9 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:40:45+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:40:45+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:40:50+00:00] done decode=16 chunk=0 +[2026-06-06T16:40:50+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:40:56+00:00] done decode=16 chunk=1 +[2026-06-06T16:40:56+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:41:01+00:00] done decode=16 chunk=2 +[2026-06-06T16:41:01+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:41:07+00:00] done decode=16 chunk=3 +[2026-06-06T16:41:07+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:41:12+00:00] done decode=16 chunk=4 +[2026-06-06T16:41:12+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:41:18+00:00] done decode=16 chunk=5 +[2026-06-06T16:41:18+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:41:23+00:00] done decode=16 chunk=6 +[2026-06-06T16:41:23+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:41:29+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 51.684858774592136 4.9840054686121995 0.06703694535552943 0.4715866016863928 0.0696670586051117 59 60 65022 64636 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 86.03860754459987 5.089128018693972 0.07158906916036084 0.5035605121850475 0.029937547500247828 0 0 56772 60526 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:41:42+00:00] done +[2026-06-06T16:41:42+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.8 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:41:42+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:41:42+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:41:47+00:00] done decode=16 chunk=0 +[2026-06-06T16:41:47+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:41:53+00:00] done decode=16 chunk=1 +[2026-06-06T16:41:53+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:41:59+00:00] done decode=16 chunk=2 +[2026-06-06T16:41:59+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:42:04+00:00] done decode=16 chunk=3 +[2026-06-06T16:42:04+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:42:10+00:00] done decode=16 chunk=4 +[2026-06-06T16:42:10+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:42:15+00:00] done decode=16 chunk=5 +[2026-06-06T16:42:15+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:42:21+00:00] done decode=16 chunk=6 +[2026-06-06T16:42:21+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:42:26+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 60.67968479746182 5.029885719291576 0.06760737142416531 0.47547961260872246 0.036718328321381756 59 64 61943 64845 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 79.6457628474405 5.0656516143225785 0.07005992808629645 0.49268066609134786 0.027790651218537753 0 0 57372 62575 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:42:39+00:00] done +[2026-06-06T16:42:39+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.8 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:42:39+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:42:39+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:42:45+00:00] done decode=16 chunk=0 +[2026-06-06T16:42:45+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:42:50+00:00] done decode=16 chunk=1 +[2026-06-06T16:42:50+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:42:56+00:00] done decode=16 chunk=2 +[2026-06-06T16:42:56+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:43:02+00:00] done decode=16 chunk=3 +[2026-06-06T16:43:02+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:43:08+00:00] done decode=16 chunk=4 +[2026-06-06T16:43:08+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:43:13+00:00] done decode=16 chunk=5 +[2026-06-06T16:43:13+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:43:19+00:00] done decode=16 chunk=6 +[2026-06-06T16:43:19+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:43:25+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 60.10263070094913 5.067019575022573 0.06866607951297146 0.4821687938440619 0.046740524420958295 60 66 62551 64719 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 86.7754526169408 5.125911203511233 0.07192335081245549 0.504992959684076 0.02788567359357804 0 0 56659 61788 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:43:38+00:00] done +[2026-06-06T16:43:38+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.8 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:43:38+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:43:38+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:43:44+00:00] done decode=16 chunk=0 +[2026-06-06T16:43:44+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:43:49+00:00] done decode=16 chunk=1 +[2026-06-06T16:43:49+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:43:55+00:00] done decode=16 chunk=2 +[2026-06-06T16:43:55+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:44:00+00:00] done decode=16 chunk=3 +[2026-06-06T16:44:00+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:44:06+00:00] done decode=16 chunk=4 +[2026-06-06T16:44:06+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:44:11+00:00] done decode=16 chunk=5 +[2026-06-06T16:44:11+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:44:16+00:00] done decode=16 chunk=6 +[2026-06-06T16:44:16+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:44:22+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 60.89392726464081 5.076946363583036 0.0719118828400173 0.49316401724058023 0.056896125563863316 58 62 63354 64732 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 93.20237118167213 5.154533333003376 0.07582174153826107 0.5199126936295669 0.030198390722220414 0 0 56641 61394 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:44:35+00:00] done +[2026-06-06T16:44:35+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.8 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:44:35+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:44:35+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:44:40+00:00] done decode=16 chunk=0 +[2026-06-06T16:44:40+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:44:46+00:00] done decode=16 chunk=1 +[2026-06-06T16:44:46+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:44:51+00:00] done decode=16 chunk=2 +[2026-06-06T16:44:51+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:44:57+00:00] done decode=16 chunk=3 +[2026-06-06T16:44:57+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:45:02+00:00] done decode=16 chunk=4 +[2026-06-06T16:45:02+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:45:07+00:00] done decode=16 chunk=5 +[2026-06-06T16:45:07+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:45:13+00:00] done decode=16 chunk=6 +[2026-06-06T16:45:13+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:45:18+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 57.769984995905496 5.0828929492462125 0.07108660736347912 0.492641040292814 0.0605386706202125 61 64 64202 64752 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 93.64787371978711 5.17563703944959 0.07552215786969434 0.5232329198661154 0.02946725951205106 0 0 56570 60949 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:45:31+00:00] done +[2026-06-06T16:45:31+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.7 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:45:31+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:45:31+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:45:37+00:00] done decode=16 chunk=0 +[2026-06-06T16:45:37+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:45:42+00:00] done decode=16 chunk=1 +[2026-06-06T16:45:42+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:45:48+00:00] done decode=16 chunk=2 +[2026-06-06T16:45:48+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:45:53+00:00] done decode=16 chunk=3 +[2026-06-06T16:45:53+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:45:59+00:00] done decode=16 chunk=4 +[2026-06-06T16:45:59+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:46:04+00:00] done decode=16 chunk=5 +[2026-06-06T16:46:04+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:46:10+00:00] done decode=16 chunk=6 +[2026-06-06T16:46:10+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:46:15+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 69.70207233381 5.135865766855551 0.07141975689516876 0.5034784889012294 0.029447152464984267 57 59 61460 64828 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 85.20949409341574 5.159956357367022 0.07334072548709013 0.5169890224777836 0.025788056391572944 0 0 58027 63130 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:46:28+00:00] done +[2026-06-06T16:46:28+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.7 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:46:28+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:46:28+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:46:33+00:00] done decode=16 chunk=0 +[2026-06-06T16:46:33+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:46:39+00:00] done decode=16 chunk=1 +[2026-06-06T16:46:39+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:46:44+00:00] done decode=16 chunk=2 +[2026-06-06T16:46:44+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:46:50+00:00] done decode=16 chunk=3 +[2026-06-06T16:46:50+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:46:55+00:00] done decode=16 chunk=4 +[2026-06-06T16:46:55+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:47:01+00:00] done decode=16 chunk=5 +[2026-06-06T16:47:01+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:47:06+00:00] done decode=16 chunk=6 +[2026-06-06T16:47:06+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:47:12+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 66.19159011596544 5.116129580407674 0.07192271543688274 0.5067486364056578 0.03910450980694267 60 63 62059 64903 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 90.78804299598454 5.161179181895141 0.07481848343511083 0.5271197307260779 0.025452388966357327 0 0 56999 62391 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:47:24+00:00] done +[2026-06-06T16:47:24+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.7 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:47:24+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:47:24+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:47:30+00:00] done decode=16 chunk=0 +[2026-06-06T16:47:30+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:47:35+00:00] done decode=16 chunk=1 +[2026-06-06T16:47:35+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:47:41+00:00] done decode=16 chunk=2 +[2026-06-06T16:47:41+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:47:46+00:00] done decode=16 chunk=3 +[2026-06-06T16:47:46+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:47:52+00:00] done decode=16 chunk=4 +[2026-06-06T16:47:52+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:47:57+00:00] done decode=16 chunk=5 +[2026-06-06T16:47:57+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:48:03+00:00] done decode=16 chunk=6 +[2026-06-06T16:48:03+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:48:08+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 68.36237080355224 5.155698714016025 0.07431390687634906 0.5134522579749919 0.047872340425531915 58 63 62518 64860 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 100.5954634875056 5.2215429291642295 0.07784257030707352 0.5378153975382031 0.026814415171143812 0 0 56575 61907 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:48:21+00:00] done +[2026-06-06T16:48:21+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.7 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:48:21+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:48:21+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:48:27+00:00] done decode=16 chunk=0 +[2026-06-06T16:48:27+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:48:32+00:00] done decode=16 chunk=1 +[2026-06-06T16:48:32+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:48:38+00:00] done decode=16 chunk=2 +[2026-06-06T16:48:38+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:48:43+00:00] done decode=16 chunk=3 +[2026-06-06T16:48:43+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:48:49+00:00] done decode=16 chunk=4 +[2026-06-06T16:48:49+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:48:54+00:00] done decode=16 chunk=5 +[2026-06-06T16:48:54+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:49:00+00:00] done decode=16 chunk=6 +[2026-06-06T16:49:00+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:49:05+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 69.65866223716107 5.171309094742462 0.07575172584207193 0.5222084942084942 0.0559064724869114 61 63 63196 64751 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 109.2556763900965 5.255902200997941 0.07992895204262877 0.5509402600788711 0.027425163361415743 0 0 56393 61367 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:49:18+00:00] done +[2026-06-06T16:49:18+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.6 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:49:18+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode16_n64 +[2026-06-06T16:49:18+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:49:24+00:00] done decode=16 chunk=0 +[2026-06-06T16:49:24+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:49:29+00:00] done decode=16 chunk=1 +[2026-06-06T16:49:29+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:49:35+00:00] done decode=16 chunk=2 +[2026-06-06T16:49:35+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:49:40+00:00] done decode=16 chunk=3 +[2026-06-06T16:49:40+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:49:46+00:00] done decode=16 chunk=4 +[2026-06-06T16:49:46+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:49:51+00:00] done decode=16 chunk=5 +[2026-06-06T16:49:51+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:49:56+00:00] done decode=16 chunk=6 +[2026-06-06T16:49:56+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:50:02+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 81.08656358254632 5.201009844925573 0.0744021940096142 0.5241360183658691 0.024251201774929128 60 62 60476 64904 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 97.33019619989669 5.216910754994746 0.07617920807698375 0.5366230734039029 0.02276384287742546 0 0 57405 63390 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:50:15+00:00] done +[2026-06-06T16:50:15+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.6 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:50:15+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode16_n64 +[2026-06-06T16:50:15+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:50:20+00:00] done decode=16 chunk=0 +[2026-06-06T16:50:20+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:50:26+00:00] done decode=16 chunk=1 +[2026-06-06T16:50:26+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:50:31+00:00] done decode=16 chunk=2 +[2026-06-06T16:50:31+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:50:37+00:00] done decode=16 chunk=3 +[2026-06-06T16:50:37+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:50:42+00:00] done decode=16 chunk=4 +[2026-06-06T16:50:42+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:50:48+00:00] done decode=16 chunk=5 +[2026-06-06T16:50:48+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:50:53+00:00] done decode=16 chunk=6 +[2026-06-06T16:50:53+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:50:59+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 80.04116043116768 5.212203343451233 0.0755115247134229 0.5294978660462536 0.031107481819302354 60 66 61124 64904 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 102.81085275851225 5.244559565410965 0.07787894678298454 0.5460511679644049 0.02356549236465335 0 0 57141 62931 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:51:11+00:00] done +[2026-06-06T16:51:11+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.6 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:51:11+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode16_n64 +[2026-06-06T16:51:11+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:51:17+00:00] done decode=16 chunk=0 +[2026-06-06T16:51:17+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:51:22+00:00] done decode=16 chunk=1 +[2026-06-06T16:51:22+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:51:28+00:00] done decode=16 chunk=2 +[2026-06-06T16:51:28+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:51:33+00:00] done decode=16 chunk=3 +[2026-06-06T16:51:33+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:51:39+00:00] done decode=16 chunk=4 +[2026-06-06T16:51:39+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:51:44+00:00] done decode=16 chunk=5 +[2026-06-06T16:51:44+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:51:50+00:00] done decode=16 chunk=6 +[2026-06-06T16:51:50+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:51:55+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 77.42595791417497 5.20392680002306 0.07624299180580371 0.5350184064199128 0.04030866859712895 60 63 61894 64924 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 106.84854897922797 5.250332759328893 0.07923090469557978 0.5559540200441868 0.024318396490722507 0 0 56936 62463 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:52:08+00:00] done +[2026-06-06T16:52:08+00:00] run steps=16 schedule=logit_normal gamma=2.0 m=-0.6 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=16 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:52:08+00:00] infer step=103000 decode=16 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode16_n64 +[2026-06-06T16:52:08+00:00] run decode=16 chunk=0 n=8 seed=123 +[2026-06-06T16:52:14+00:00] done decode=16 chunk=0 +[2026-06-06T16:52:14+00:00] run decode=16 chunk=1 n=8 seed=124 +[2026-06-06T16:52:19+00:00] done decode=16 chunk=1 +[2026-06-06T16:52:19+00:00] run decode=16 chunk=2 n=8 seed=125 +[2026-06-06T16:52:25+00:00] done decode=16 chunk=2 +[2026-06-06T16:52:25+00:00] run decode=16 chunk=3 n=8 seed=126 +[2026-06-06T16:52:30+00:00] done decode=16 chunk=3 +[2026-06-06T16:52:30+00:00] run decode=16 chunk=4 n=8 seed=127 +[2026-06-06T16:52:36+00:00] done decode=16 chunk=4 +[2026-06-06T16:52:36+00:00] run decode=16 chunk=5 n=8 seed=128 +[2026-06-06T16:52:41+00:00] done decode=16 chunk=5 +[2026-06-06T16:52:41+00:00] run decode=16 chunk=6 n=8 seed=129 +[2026-06-06T16:52:47+00:00] done decode=16 chunk=6 +[2026-06-06T16:52:47+00:00] run decode=16 chunk=7 n=8 seed=130 +[2026-06-06T16:52:53+00:00] done decode=16 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode16_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 75.25187919775422 5.227538831324605 0.07938218943931998 0.5407003603437125 0.05013936155468979 58 62 62536 64939 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode16_n64/sc1p0 +sc1p0 pre_eos 113.60202459087829 5.302007076422275 0.08332121094570787 0.5673369108424388 0.02519840307747014 0 0 56384 61869 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode16_n64/sc1p0 +[2026-06-06T16:53:05+00:00] done +[2026-06-06T16:53:05+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.1 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:53:05+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode32_n64 +[2026-06-06T16:53:05+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:53:12+00:00] done decode=32 chunk=0 +[2026-06-06T16:53:12+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:53:19+00:00] done decode=32 chunk=1 +[2026-06-06T16:53:19+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:53:26+00:00] done decode=32 chunk=2 +[2026-06-06T16:53:26+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:53:33+00:00] done decode=32 chunk=3 +[2026-06-06T16:53:33+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:53:40+00:00] done decode=32 chunk=4 +[2026-06-06T16:53:40+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:53:47+00:00] done decode=32 chunk=5 +[2026-06-06T16:53:47+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:53:54+00:00] done decode=32 chunk=6 +[2026-06-06T16:53:54+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:54:01+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.166337890847695 4.448900463028334 0.03587265665616685 0.27689343547869166 0.08065995902015352 60 62 67277 65398 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 17.24407669063869 4.535513797805912 0.038781346602086195 0.2992792435363354 0.03666539930239862 0 0 57442 60493 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:54:14+00:00] done +[2026-06-06T16:54:14+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.1 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:54:14+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode32_n64 +[2026-06-06T16:54:14+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:54:21+00:00] done decode=32 chunk=0 +[2026-06-06T16:54:21+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:54:28+00:00] done decode=32 chunk=1 +[2026-06-06T16:54:28+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:54:35+00:00] done decode=32 chunk=2 +[2026-06-06T16:54:35+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:54:42+00:00] done decode=32 chunk=3 +[2026-06-06T16:54:42+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:54:49+00:00] done decode=32 chunk=4 +[2026-06-06T16:54:49+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:54:56+00:00] done decode=32 chunk=5 +[2026-06-06T16:54:56+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:55:03+00:00] done decode=32 chunk=6 +[2026-06-06T16:55:03+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:55:10+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.717059679683208 4.4947770257902455 0.03692505039398937 0.2838446619733366 0.08822002321177692 61 61 67929 65484 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 19.476620525650567 4.615530722321262 0.040438163726064055 0.3108004147573335 0.033982774479471525 0 0 56526 59795 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:55:23+00:00] done +[2026-06-06T16:55:23+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.1 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:55:23+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode32_n64 +[2026-06-06T16:55:23+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:55:30+00:00] done decode=32 chunk=0 +[2026-06-06T16:55:30+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:55:37+00:00] done decode=32 chunk=1 +[2026-06-06T16:55:37+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:55:44+00:00] done decode=32 chunk=2 +[2026-06-06T16:55:44+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:55:51+00:00] done decode=32 chunk=3 +[2026-06-06T16:55:51+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:55:58+00:00] done decode=32 chunk=4 +[2026-06-06T16:55:58+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:56:05+00:00] done decode=32 chunk=5 +[2026-06-06T16:56:05+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:56:12+00:00] done decode=32 chunk=6 +[2026-06-06T16:56:12+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:56:18+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.160048883349862 4.4427050548744775 0.03824802336784475 0.28752982198568544 0.09550536022878466 61 62 68481 65389 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 19.035768590173294 4.5762564775530885 0.0421199770958941 0.316604072283881 0.032570985887028864 0 0 56431 59378 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:56:32+00:00] done +[2026-06-06T16:56:32+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.1 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:56:32+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode32_n64 +[2026-06-06T16:56:32+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:56:39+00:00] done decode=32 chunk=0 +[2026-06-06T16:56:39+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:56:46+00:00] done decode=32 chunk=1 +[2026-06-06T16:56:46+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:56:53+00:00] done decode=32 chunk=2 +[2026-06-06T16:56:53+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:56:59+00:00] done decode=32 chunk=3 +[2026-06-06T16:56:59+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:57:07+00:00] done decode=32 chunk=4 +[2026-06-06T16:57:07+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:57:14+00:00] done decode=32 chunk=5 +[2026-06-06T16:57:14+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:57:20+00:00] done decode=32 chunk=6 +[2026-06-06T16:57:20+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:57:27+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.460643735202217 4.419368513602272 0.03938523588364703 0.2990604231915056 0.10330603764360792 62 63 69251 65456 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 21.00777055353242 4.586715953323252 0.04389280484898015 0.3332538777178077 0.03263867606497089 0 0 55770 58734 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p1_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p1_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:57:41+00:00] done +[2026-06-06T16:57:41+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.0 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:57:41+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode32_n64 +[2026-06-06T16:57:41+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:57:48+00:00] done decode=32 chunk=0 +[2026-06-06T16:57:48+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:57:55+00:00] done decode=32 chunk=1 +[2026-06-06T16:57:55+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:58:02+00:00] done decode=32 chunk=2 +[2026-06-06T16:58:02+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:58:08+00:00] done decode=32 chunk=3 +[2026-06-06T16:58:08+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:58:15+00:00] done decode=32 chunk=4 +[2026-06-06T16:58:15+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:58:22+00:00] done decode=32 chunk=5 +[2026-06-06T16:58:22+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:58:29+00:00] done decode=32 chunk=6 +[2026-06-06T16:58:29+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:58:36+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.487511320195035 4.582127945344643 0.040984607867090156 0.29764686120909495 0.07595284632299047 63 63 66809 65488 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 17.918778792059822 4.679928251813925 0.044385645774764344 0.3223304503133837 0.0322970067802216 0 0 56740 60470 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:58:49+00:00] done +[2026-06-06T16:58:49+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.0 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:58:49+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode32_n64 +[2026-06-06T16:58:49+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T16:58:56+00:00] done decode=32 chunk=0 +[2026-06-06T16:58:56+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T16:59:03+00:00] done decode=32 chunk=1 +[2026-06-06T16:59:03+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T16:59:10+00:00] done decode=32 chunk=2 +[2026-06-06T16:59:10+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T16:59:17+00:00] done decode=32 chunk=3 +[2026-06-06T16:59:17+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T16:59:24+00:00] done decode=32 chunk=4 +[2026-06-06T16:59:24+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T16:59:31+00:00] done decode=32 chunk=5 +[2026-06-06T16:59:31+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T16:59:38+00:00] done decode=32 chunk=6 +[2026-06-06T16:59:38+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T16:59:45+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 12.90511536801487 4.596315649450101 0.04058471314229853 0.30129683657415185 0.07745768925276471 63 63 67287 65468 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 18.812166598884108 4.701244999603159 0.0440287006810612 0.32684850694329365 0.03221369744974895 0 0 57011 60347 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T16:59:59+00:00] done +[2026-06-06T16:59:59+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.0 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T16:59:59+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode32_n64 +[2026-06-06T16:59:59+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:00:06+00:00] done decode=32 chunk=0 +[2026-06-06T17:00:06+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:00:13+00:00] done decode=32 chunk=1 +[2026-06-06T17:00:13+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:00:21+00:00] done decode=32 chunk=2 +[2026-06-06T17:00:21+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:00:27+00:00] done decode=32 chunk=3 +[2026-06-06T17:00:27+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:00:34+00:00] done decode=32 chunk=4 +[2026-06-06T17:00:34+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:00:41+00:00] done decode=32 chunk=5 +[2026-06-06T17:00:41+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:00:48+00:00] done decode=32 chunk=6 +[2026-06-06T17:00:48+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:00:55+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 13.37281852787122 4.5843499795427345 0.04285408512720157 0.31433944379042 0.08729818982387476 63 63 67573 65408 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 20.596437390533897 4.709408568736831 0.04688231752191075 0.3438707411310151 0.03226399946477554 0 0 56303 59788 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:01:08+00:00] done +[2026-06-06T17:01:08+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-1.0 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:01:08+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode32_n64 +[2026-06-06T17:01:08+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:01:15+00:00] done decode=32 chunk=0 +[2026-06-06T17:01:15+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:01:22+00:00] done decode=32 chunk=1 +[2026-06-06T17:01:22+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:01:29+00:00] done decode=32 chunk=2 +[2026-06-06T17:01:29+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:01:36+00:00] done decode=32 chunk=3 +[2026-06-06T17:01:36+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:01:43+00:00] done decode=32 chunk=4 +[2026-06-06T17:01:43+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:01:50+00:00] done decode=32 chunk=5 +[2026-06-06T17:01:50+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:01:57+00:00] done decode=32 chunk=6 +[2026-06-06T17:01:57+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:02:04+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 13.275588750739667 4.568902751559927 0.04452463227424238 0.32007155853886027 0.09817742576679613 63 63 68550 65402 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 21.769705224292117 4.729389076074023 0.04926325049483176 0.35411943833530707 0.032159834886907684 0 0 55940 59111 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn1p0_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn1p0_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:02:17+00:00] done +[2026-06-06T17:02:17+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.9 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:02:17+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode32_n64 +[2026-06-06T17:02:17+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:02:24+00:00] done decode=32 chunk=0 +[2026-06-06T17:02:24+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:02:31+00:00] done decode=32 chunk=1 +[2026-06-06T17:02:31+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:02:38+00:00] done decode=32 chunk=2 +[2026-06-06T17:02:38+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:02:45+00:00] done decode=32 chunk=3 +[2026-06-06T17:02:45+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:02:52+00:00] done decode=32 chunk=4 +[2026-06-06T17:02:52+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:02:59+00:00] done decode=32 chunk=5 +[2026-06-06T17:02:59+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:03:05+00:00] done decode=32 chunk=6 +[2026-06-06T17:03:05+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:03:12+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 13.923779353409087 4.701108291953996 0.04756953470236548 0.32771644392794885 0.0654444257557455 64 64 65331 65399 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 19.26028806078032 4.782075654031104 0.05093934777979788 0.3509467339317303 0.031595498992678495 0 0 56608 61053 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:03:25+00:00] done +[2026-06-06T17:03:25+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.9 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:03:25+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode32_n64 +[2026-06-06T17:03:25+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:03:32+00:00] done decode=32 chunk=0 +[2026-06-06T17:03:32+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:03:39+00:00] done decode=32 chunk=1 +[2026-06-06T17:03:39+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:03:46+00:00] done decode=32 chunk=2 +[2026-06-06T17:03:46+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:03:53+00:00] done decode=32 chunk=3 +[2026-06-06T17:03:53+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:04:00+00:00] done decode=32 chunk=4 +[2026-06-06T17:04:00+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:04:07+00:00] done decode=32 chunk=5 +[2026-06-06T17:04:07+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:04:14+00:00] done decode=32 chunk=6 +[2026-06-06T17:04:14+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:04:21+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 14.127519770290512 4.700041345846318 0.04736552230585644 0.3359698912228631 0.07623462456398017 64 64 66648 65364 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 20.83827505558188 4.808667108576711 0.051312233698625594 0.36403276079315605 0.032412089460682726 0 0 56518 60317 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:04:34+00:00] done +[2026-06-06T17:04:34+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.9 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:04:34+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode32_n64 +[2026-06-06T17:04:34+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:04:41+00:00] done decode=32 chunk=0 +[2026-06-06T17:04:41+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:04:48+00:00] done decode=32 chunk=1 +[2026-06-06T17:04:48+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:04:55+00:00] done decode=32 chunk=2 +[2026-06-06T17:04:55+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:05:02+00:00] done decode=32 chunk=3 +[2026-06-06T17:05:02+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:05:09+00:00] done decode=32 chunk=4 +[2026-06-06T17:05:09+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:05:16+00:00] done decode=32 chunk=5 +[2026-06-06T17:05:16+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:05:23+00:00] done decode=32 chunk=6 +[2026-06-06T17:05:23+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:05:30+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 14.13749345659437 4.679216453008245 0.04442849288288839 0.3309965635738832 0.08334351518113507 64 64 67484 65476 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 21.759537838120337 4.80854025815629 0.048503043949628885 0.36142709410548085 0.031823867900925694 0 0 56409 59955 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:05:43+00:00] done +[2026-06-06T17:05:43+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.9 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:05:43+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode32_n64 +[2026-06-06T17:05:43+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:05:50+00:00] done decode=32 chunk=0 +[2026-06-06T17:05:50+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:05:57+00:00] done decode=32 chunk=1 +[2026-06-06T17:05:57+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:06:04+00:00] done decode=32 chunk=2 +[2026-06-06T17:06:04+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:06:11+00:00] done decode=32 chunk=3 +[2026-06-06T17:06:11+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:06:18+00:00] done decode=32 chunk=4 +[2026-06-06T17:06:18+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:06:25+00:00] done decode=32 chunk=5 +[2026-06-06T17:06:25+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:06:31+00:00] done decode=32 chunk=6 +[2026-06-06T17:06:31+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:06:38+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 14.152533275229109 4.664112557597337 0.04606238442376171 0.3355544687614623 0.08896122751516819 64 65 67687 65433 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 22.62713709509241 4.80797608110831 0.050600386262490554 0.36866854762864437 0.03147199596943488 0 0 55879 59545 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p9_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p9_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:06:52+00:00] done +[2026-06-06T17:06:52+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.8 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:06:52+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode32_n64 +[2026-06-06T17:06:52+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:06:59+00:00] done decode=32 chunk=0 +[2026-06-06T17:06:59+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:07:06+00:00] done decode=32 chunk=1 +[2026-06-06T17:07:06+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:07:12+00:00] done decode=32 chunk=2 +[2026-06-06T17:07:12+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:07:19+00:00] done decode=32 chunk=3 +[2026-06-06T17:07:19+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:07:26+00:00] done decode=32 chunk=4 +[2026-06-06T17:07:26+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:07:33+00:00] done decode=32 chunk=5 +[2026-06-06T17:07:33+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:07:40+00:00] done decode=32 chunk=6 +[2026-06-06T17:07:40+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:07:47+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 16.029034385732665 4.793036914820029 0.05006202238931683 0.35324205948114795 0.05534541110889907 64 64 64755 65299 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 21.175572978033298 4.853487596698464 0.05304923566258804 0.37428798623845766 0.02732790237252929 0 0 57364 61622 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:08:00+00:00] done +[2026-06-06T17:08:00+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.8 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:08:00+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode32_n64 +[2026-06-06T17:08:00+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:08:07+00:00] done decode=32 chunk=0 +[2026-06-06T17:08:07+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:08:14+00:00] done decode=32 chunk=1 +[2026-06-06T17:08:14+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:08:21+00:00] done decode=32 chunk=2 +[2026-06-06T17:08:21+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:08:28+00:00] done decode=32 chunk=3 +[2026-06-06T17:08:28+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:08:35+00:00] done decode=32 chunk=4 +[2026-06-06T17:08:35+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:08:42+00:00] done decode=32 chunk=5 +[2026-06-06T17:08:42+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:08:49+00:00] done decode=32 chunk=6 +[2026-06-06T17:08:49+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:08:56+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 15.585455748121294 4.796987829677374 0.051144792989003256 0.352925881741152 0.06942171512472661 63 63 65722 65383 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 22.237355184557735 4.894143236079336 0.0549241179948755 0.37898920881034115 0.0296629656395769 0 0 56691 60884 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:09:09+00:00] done +[2026-06-06T17:09:09+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.8 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:09:09+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode32_n64 +[2026-06-06T17:09:09+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:09:16+00:00] done decode=32 chunk=0 +[2026-06-06T17:09:16+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:09:23+00:00] done decode=32 chunk=1 +[2026-06-06T17:09:23+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:09:30+00:00] done decode=32 chunk=2 +[2026-06-06T17:09:30+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:09:37+00:00] done decode=32 chunk=3 +[2026-06-06T17:09:37+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:09:44+00:00] done decode=32 chunk=4 +[2026-06-06T17:09:44+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:09:51+00:00] done decode=32 chunk=5 +[2026-06-06T17:09:51+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:09:57+00:00] done decode=32 chunk=6 +[2026-06-06T17:09:57+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:10:04+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 15.283454213495633 4.770059389164496 0.050001526764176005 0.35236728399774037 0.0750557268924242 64 64 66898 65498 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 22.667229850020483 4.881033897328713 0.054099606728576624 0.38131434142472365 0.03127994976701147 0 0 56904 60518 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:10:17+00:00] done +[2026-06-06T17:10:17+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.8 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:10:18+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode32_n64 +[2026-06-06T17:10:18+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:10:25+00:00] done decode=32 chunk=0 +[2026-06-06T17:10:25+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:10:31+00:00] done decode=32 chunk=1 +[2026-06-06T17:10:31+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:10:38+00:00] done decode=32 chunk=2 +[2026-06-06T17:10:38+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:10:45+00:00] done decode=32 chunk=3 +[2026-06-06T17:10:45+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:10:52+00:00] done decode=32 chunk=4 +[2026-06-06T17:10:52+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:10:59+00:00] done decode=32 chunk=5 +[2026-06-06T17:10:59+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:11:06+00:00] done decode=32 chunk=6 +[2026-06-06T17:11:06+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:11:13+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 15.862170115584636 4.7581637174086815 0.04992819384605983 0.35877652666799076 0.08326458276041189 64 64 67121 65454 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 25.063652702491858 4.895620411820716 0.0545045045045045 0.3917315937870168 0.0322655989322656 0 0 56059 59940 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p8_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p8_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:11:26+00:00] done +[2026-06-06T17:11:26+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.7 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:11:26+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode32_n64 +[2026-06-06T17:11:26+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:11:33+00:00] done decode=32 chunk=0 +[2026-06-06T17:11:33+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:11:40+00:00] done decode=32 chunk=1 +[2026-06-06T17:11:40+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:11:47+00:00] done decode=32 chunk=2 +[2026-06-06T17:11:47+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:11:54+00:00] done decode=32 chunk=3 +[2026-06-06T17:11:54+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:12:01+00:00] done decode=32 chunk=4 +[2026-06-06T17:12:01+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:12:08+00:00] done decode=32 chunk=5 +[2026-06-06T17:12:08+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:12:15+00:00] done decode=32 chunk=6 +[2026-06-06T17:12:15+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:12:22+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 17.594936116216314 4.887315286074233 0.05435281166865425 0.3756593532604541 0.04721279393327829 63 63 64039 65406 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 22.440210035242004 4.939221450031322 0.0570992611628654 0.3946256766090043 0.026389335046578862 0 0 57710 62260 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:12:35+00:00] done +[2026-06-06T17:12:35+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.7 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:12:35+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode32_n64 +[2026-06-06T17:12:35+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:12:42+00:00] done decode=32 chunk=0 +[2026-06-06T17:12:42+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:12:49+00:00] done decode=32 chunk=1 +[2026-06-06T17:12:49+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:12:56+00:00] done decode=32 chunk=2 +[2026-06-06T17:12:56+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:13:03+00:00] done decode=32 chunk=3 +[2026-06-06T17:13:03+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:13:10+00:00] done decode=32 chunk=4 +[2026-06-06T17:13:10+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:13:16+00:00] done decode=32 chunk=5 +[2026-06-06T17:13:16+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:13:23+00:00] done decode=32 chunk=6 +[2026-06-06T17:13:23+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:13:30+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 18.042091834633876 4.873580868503792 0.05585248939461231 0.38160071061014456 0.059956812717276445 64 65 64386 65297 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 25.112357250245818 4.952618406475011 0.05946148702643639 0.40632135168634614 0.028377122168403542 0 0 56391 61317 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:13:43+00:00] done +[2026-06-06T17:13:43+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.7 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:13:43+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode32_n64 +[2026-06-06T17:13:43+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:13:50+00:00] done decode=32 chunk=0 +[2026-06-06T17:13:50+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:13:57+00:00] done decode=32 chunk=1 +[2026-06-06T17:13:57+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:14:04+00:00] done decode=32 chunk=2 +[2026-06-06T17:14:04+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:14:11+00:00] done decode=32 chunk=3 +[2026-06-06T17:14:11+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:14:18+00:00] done decode=32 chunk=4 +[2026-06-06T17:14:18+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:14:25+00:00] done decode=32 chunk=5 +[2026-06-06T17:14:25+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:14:32+00:00] done decode=32 chunk=6 +[2026-06-06T17:14:32+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:14:39+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 17.38305180588535 4.860753242967672 0.05470111603730316 0.3810790563989665 0.0705090964684299 64 64 66047 65410 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 25.66969738292471 4.967187637586616 0.05889617018474001 0.4103699800767293 0.03013139263015774 0 0 56661 60734 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:14:52+00:00] done +[2026-06-06T17:14:52+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.7 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:14:52+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode32_n64 +[2026-06-06T17:14:52+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:14:59+00:00] done decode=32 chunk=0 +[2026-06-06T17:14:59+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:15:06+00:00] done decode=32 chunk=1 +[2026-06-06T17:15:06+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:15:13+00:00] done decode=32 chunk=2 +[2026-06-06T17:15:13+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:15:20+00:00] done decode=32 chunk=3 +[2026-06-06T17:15:20+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:15:27+00:00] done decode=32 chunk=4 +[2026-06-06T17:15:27+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:15:34+00:00] done decode=32 chunk=5 +[2026-06-06T17:15:34+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:15:41+00:00] done decode=32 chunk=6 +[2026-06-06T17:15:41+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:15:48+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 17.4085980979728 4.844691979736777 0.055100669090464696 0.381830402835276 0.07483731019522777 64 64 66832 65462 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 26.39894298763466 4.962739321632926 0.0596042909800162 0.4131045654401798 0.03132283178234351 0 0 56874 60499 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p7_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p7_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:16:01+00:00] done +[2026-06-06T17:16:01+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.6 s=0.5 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:16:01+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode32_n64 +[2026-06-06T17:16:01+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:16:08+00:00] done decode=32 chunk=0 +[2026-06-06T17:16:08+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:16:15+00:00] done decode=32 chunk=1 +[2026-06-06T17:16:15+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:16:22+00:00] done decode=32 chunk=2 +[2026-06-06T17:16:22+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:16:29+00:00] done decode=32 chunk=3 +[2026-06-06T17:16:29+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:16:36+00:00] done decode=32 chunk=4 +[2026-06-06T17:16:36+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:16:43+00:00] done decode=32 chunk=5 +[2026-06-06T17:16:43+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:16:49+00:00] done decode=32 chunk=6 +[2026-06-06T17:16:49+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:16:56+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 20.78915077992427 4.960328336882147 0.05863291758376131 0.40014979441480825 0.03935864514551235 64 64 62958 65424 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 25.67813136859051 5.0015138175604354 0.061081468503623476 0.4169215086646279 0.026933184677868916 0 0 57637 62785 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p5_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p5_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:17:09+00:00] done +[2026-06-06T17:17:09+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.6 s=0.6 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:17:09+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode32_n64 +[2026-06-06T17:17:09+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:17:16+00:00] done decode=32 chunk=0 +[2026-06-06T17:17:16+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:17:23+00:00] done decode=32 chunk=1 +[2026-06-06T17:17:23+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:17:30+00:00] done decode=32 chunk=2 +[2026-06-06T17:17:30+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:17:37+00:00] done decode=32 chunk=3 +[2026-06-06T17:17:37+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:17:44+00:00] done decode=32 chunk=4 +[2026-06-06T17:17:44+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:17:51+00:00] done decode=32 chunk=5 +[2026-06-06T17:17:51+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:17:58+00:00] done decode=32 chunk=6 +[2026-06-06T17:17:58+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:18:05+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 21.728295821199843 4.964750353709601 0.059043073910915324 0.4072657743785851 0.048519334312285854 64 65 63655 65376 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 28.70082062686809 5.02204049656462 0.062102705225381805 0.4284173935434034 0.028693735013437617 0 0 57145 62139 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p6_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p6_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:18:18+00:00] done +[2026-06-06T17:18:18+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.6 s=0.7 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:18:18+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode32_n64 +[2026-06-06T17:18:18+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:18:25+00:00] done decode=32 chunk=0 +[2026-06-06T17:18:25+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:18:32+00:00] done decode=32 chunk=1 +[2026-06-06T17:18:32+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:18:38+00:00] done decode=32 chunk=2 +[2026-06-06T17:18:38+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:18:45+00:00] done decode=32 chunk=3 +[2026-06-06T17:18:45+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:18:52+00:00] done decode=32 chunk=4 +[2026-06-06T17:18:52+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:18:59+00:00] done decode=32 chunk=5 +[2026-06-06T17:18:59+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:19:07+00:00] done decode=32 chunk=6 +[2026-06-06T17:19:07+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:19:13+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 20.595399485949486 4.93816281186297 0.060398477040933346 0.40629682864919414 0.06094894417345831 64 64 64658 65399 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 29.418144588494474 5.025569151769037 0.06436942737452933 0.43307035274173566 0.03136155438556456 0 0 56526 61349 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p7_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p7_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:19:26+00:00] done +[2026-06-06T17:19:26+00:00] run steps=32 schedule=logit_normal gamma=2.0 m=-0.6 s=0.8 tag=hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8 +checkpoint=/e2e-data/evad-tech-vla/huangzhijian5/tmp/lta/20260604/owt_t5_elftokenized_full_len1024_C1_toV_power2_d768_l12_h12_gbs512_16gpu_50ep_lr3e4_elfopt_t5embed_unfixed_adaln_rope_noabspos_stateprobadd_selfcond_ce_fast_20260605_024231/step_103000.pt +use_ema=0 +step=103000 +decode_steps=32 +n=64 chunk_n=8 gpu=3 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606 +[2026-06-06T17:19:26+00:00] infer step=103000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode32_n64 +[2026-06-06T17:19:26+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-06T17:19:33+00:00] done decode=32 chunk=0 +[2026-06-06T17:19:33+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-06T17:19:40+00:00] done decode=32 chunk=1 +[2026-06-06T17:19:40+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-06T17:19:47+00:00] done decode=32 chunk=2 +[2026-06-06T17:19:47+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-06T17:19:54+00:00] done decode=32 chunk=3 +[2026-06-06T17:19:54+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-06T17:20:01+00:00] done decode=32 chunk=4 +[2026-06-06T17:20:01+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-06T17:20:08+00:00] done decode=32 chunk=5 +[2026-06-06T17:20:08+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-06T17:20:15+00:00] done decode=32 chunk=6 +[2026-06-06T17:20:15+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-06T17:20:22+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 19.499581056872177 4.929030334351387 0.05859506406923759 0.40382586357172345 0.0708737270252913 64 64 66077 65398 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 29.457188070070497 5.043321855922785 0.06311471358671478 0.4350390457675706 0.029143807970477272 0 0 56639 60699 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260606/hzj20260604_adaln_rope_noabspos_C1toV_power2_decode_gamma_finesweep_ln_mn0p6_s0p8_step103000_dgamma2p0_tschedlogit_normal_mn0p6_s0p8_sc1p0_decode32_n64/sc1p0 +[2026-06-06T17:20:35+00:00] done +[2026-06-06T17:20:35+00:00] C1toV power2 decode gamma fine sweep done diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245.log new file mode 100644 index 0000000000000000000000000000000000000000..f2cfcf5a023fe97756e1d161a9185749abd9b398 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245.log @@ -0,0 +1,8080 @@ +[data] mode=elf_hfds direct-tokenized-fast path=/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5 +[data] t5_tokenized_arrow_shards=75 rows=9737184 steps_per_epoch_gbs512=19018 epochs=50 total_steps=950900 +[bridge] C=1->1024 schedule=exp +[state-probadd] train: p_u=softmax(f(state,t)); state_in=state or (state+scale*p_u)/(1+scale); CE(f(state_in,t), y) +[time] sample_dist=uniform logit_mean=-1.5 logit_std=0.6 +[embed] token_embed_init=t5_shared freeze_token_embed=0 freeze_t5_encoder=1 +[arch] time_conditioning=prefix time_tokens=4 use_abs_pos=1 state_self_cond_normalize=1 +[dist] nnodes=1 nproc_per_node=8 master=127.0.0.1:33041 node_rank=0 +[batch] batch_size=16 grad_accum=4 +[ema] decay=0.9999 start_steps=0 +W0606 14:42:49.538000 10383 torch/distributed/run.py:792] +W0606 14:42:49.538000 10383 torch/distributed/run.py:792] ***************************************** +W0606 14:42:49.538000 10383 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0606 14:42:49.538000 10383 torch/distributed/run.py:792] ***************************************** +[W606 14:42:52.835053809 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.836904071 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.840701317 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.850027099 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.850721847 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.850787851 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.851667324 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W606 14:42:52.852039821 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[ema] initialized from current model weights +t-20260606223951-29b25-worker-0:10450:10450 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10450:10450 [0] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10450:10450 [0] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10450:10450 [0] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10450:10450 [0] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10455:10455 [5] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10455:10455 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10451:10451 [1] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10451:10451 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10451:10451 [1] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10455:10455 [5] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10451:10451 [1] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10455:10455 [5] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10454:10454 [4] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10454:10454 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10453:10453 [3] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10457:10457 [7] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10453:10453 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10457:10457 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10452:10452 [2] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10452:10452 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10456:10456 [6] NCCL INFO cudaDriverVersion 12080 +t-20260606223951-29b25-worker-0:10456:10456 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10451:10451 [1] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10455:10455 [5] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10454:10454 [4] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10454:10454 [4] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10457:10457 [7] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10453:10453 [3] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10457:10457 [7] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10452:10452 [2] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10453:10453 [3] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10452:10452 [2] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10456:10456 [6] NCCL INFO Bootstrap: Using eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10456:10456 [6] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260606223951-29b25-worker-0:10454:10454 [4] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10457:10457 [7] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10452:10452 [2] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10453:10453 [3] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10456:10456 [6] NCCL INFO Comm config Blocking set to 1 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO P2P plugin v9 IBext_v9 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.126<0> +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Using network IBext_v9 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO ncclCommInitRankConfig comm 0xa070080 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO ncclCommInitRankConfig comm 0xb194090 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO ncclCommInitRankConfig comm 0xb3b8290 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO ncclCommInitRankConfig comm 0xb880ab0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO ncclCommInitRankConfig comm 0x9e94a30 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO ncclCommInitRankConfig comm 0xb57f970 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO ncclCommInitRankConfig comm 0xa9fcfd0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO ncclCommInitRankConfig comm 0xb306d00 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x6725cbc81b734c51 - Init START +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO RAS client listening socket at ::1<28028> +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Bootstrap timings total 0.039710 (create 0.000016, send 0.000061, recv 0.000089, ring 0.004122, delay 0.000001) +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Bootstrap timings total 0.042150 (create 0.000022, send 0.000085, recv 0.000076, ring 0.026844, delay 0.000001) +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Bootstrap timings total 0.058833 (create 0.000021, send 0.000072, recv 0.016740, ring 0.026905, delay 0.000001) +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Bootstrap timings total 0.065005 (create 0.000020, send 0.000085, recv 0.006206, ring 0.039232, delay 0.000001) +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Bootstrap timings total 0.069573 (create 0.000026, send 0.000075, recv 0.026430, ring 0.027898, delay 0.000001) +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Bootstrap timings total 0.068929 (create 0.000038, send 0.000085, recv 0.014445, ring 0.027867, delay 0.000001) +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Bootstrap timings total 0.044188 (create 0.000020, send 0.000064, recv 0.000103, ring 0.027893, delay 0.000001) +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Bootstrap timings total 0.063886 (create 0.000021, send 0.000063, recv 0.000065, ring 0.026868, delay 0.000001) +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO NVLS multicast support is available on dev 4 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO NVLS multicast support is available on dev 5 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO NVLS multicast support is available on dev 3 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO NVLS multicast support is available on dev 7 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO NVLS multicast support is available on dev 2 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO NVLS multicast support is available on dev 1 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO NVLS multicast support is available on dev 6 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO NVLS multicast support is available on dev 0 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO comm 0xb3b8290 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO comm 0x9e94a30 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO comm 0xb57f970 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO comm 0xb194090 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO comm 0xa9fcfd0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO comm 0xa070080 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO comm 0xb880ab0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO comm 0xb306d00 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO P2P Chunksize set to 524288 +t-20260606223951-29b25-worker-0:10451:10658 [1] NCCL INFO [Proxy Service] Device 1 CPU core 8 +t-20260606223951-29b25-worker-0:10451:10659 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 10 +t-20260606223951-29b25-worker-0:10455:10660 [5] NCCL INFO [Proxy Service] Device 5 CPU core 146 +t-20260606223951-29b25-worker-0:10455:10661 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 148 +t-20260606223951-29b25-worker-0:10454:10662 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92 +t-20260606223951-29b25-worker-0:10454:10663 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94 +t-20260606223951-29b25-worker-0:10457:10664 [7] NCCL INFO [Proxy Service] Device 7 CPU core 142 +t-20260606223951-29b25-worker-0:10457:10665 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 146 +t-20260606223951-29b25-worker-0:10456:10666 [6] NCCL INFO [Proxy Service] Device 6 CPU core 104 +t-20260606223951-29b25-worker-0:10456:10667 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 106 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0 +t-20260606223951-29b25-worker-0:10452:10669 [2] NCCL INFO [Proxy Service] Device 2 CPU core 2 +t-20260606223951-29b25-worker-0:10450:10668 [0] NCCL INFO [Proxy Service] Device 0 CPU core 66 +t-20260606223951-29b25-worker-0:10450:10671 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 69 +t-20260606223951-29b25-worker-0:10452:10672 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4 +t-20260606223951-29b25-worker-0:10453:10673 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 4 +t-20260606223951-29b25-worker-0:10453:10670 [3] NCCL INFO [Proxy Service] Device 3 CPU core 2 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO CC Off, workFifoBytes 1048576 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO ncclCommInitRankConfig comm 0xb306d00 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10453:10561 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.17 (kernels 0.22, alloc 1.08, bootstrap 0.04, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO ncclCommInitRankConfig comm 0xa9fcfd0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO ncclCommInitRankConfig comm 0xb3b8290 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO ncclCommInitRankConfig comm 0xb194090 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10456:10562 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.17 (kernels 0.22, alloc 1.08, bootstrap 0.04, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260606223951-29b25-worker-0:10451:10556 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.17 (kernels 0.19, alloc 1.09, bootstrap 0.07, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO ncclCommInitRankConfig comm 0xb880ab0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO ncclCommInitRankConfig comm 0x9e94a30 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10457:10559 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.17 (kernels 0.19, alloc 1.08, bootstrap 0.07, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO ncclCommInitRankConfig comm 0xb57f970 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10454:10558 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.17 (kernels 0.20, alloc 1.08, bootstrap 0.06, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260606223951-29b25-worker-0:10452:10560 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.17 (kernels 0.19, alloc 1.09, bootstrap 0.06, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260606223951-29b25-worker-0:10450:10555 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.19 (kernels 0.22, alloc 1.09, bootstrap 0.04, allgathers 0.03, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO ncclCommInitRankConfig comm 0xa070080 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x6725cbc81b734c51 - Init COMPLETE +t-20260606223951-29b25-worker-0:10455:10557 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.18 (kernels 0.19, alloc 1.08, bootstrap 0.07, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260606223951-29b25-worker-0:10456:10674 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10452:10679 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10451:10680 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10455:10681 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10454:10676 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10453:10675 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10457:10677 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260606223951-29b25-worker-0:10450:10678 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +[rank2]:[W606 14:42:57.039376075 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank0]:[W606 14:42:57.039410139 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank1]:[W606 14:42:57.039427772 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank5]:[W606 14:42:57.039514884 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank7]:[W606 14:42:57.039520986 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank3]:[W606 14:42:57.039553743 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank4]:[W606 14:42:57.039590094 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank6]:[W606 14:42:57.039631279 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +Muon: 58 2D params; Nesterov-AdamW: 78 other params +Muon: 58 2D params; Nesterov-AdamW: 78 other params +Muon: 58 2D params; Nesterov-AdamW: 78 other params +Muon: 58 2D params; Nesterov-AdamW: 78 other params +{ + "data_mode": "elf_hfds", + "cache_path": "cache/owt_t5_payload1022_appendeos1.pt", + "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5", + "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json", + "text_column": "text", + "pack_len": 1023, + "append_eos": 1, + "num_workers": 4, + "shuffle_buffer": 8192, + "reject_txt": "cache/online_rejected.txt", + "out_dir": "runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245", + "subset_size": 0, + "resume": "", + "steps": 950900, + "batch_size": 16, + "grad_accum": 4, + "lr": 0.0006, + "blr": 0.001, + "min_lr": 0.0, + "lr_schedule": "constant", + "warmup_steps": 200, + "warmup_epochs": -1.0, + "optimizer": "muon", + "weight_decay": 0.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "grad_clip": 1.0, + "ema_decay": 0.9999, + "ema_start_steps": 0, + "log_every": 50, + "save_every": 1000, + "dim": 768, + "layers": 12, + "heads": 12, + "mlp_dim": 3072, + "time_tokens": 4, + "time_conditioning": "prefix", + "use_abs_pos": 1, + "time_sample_dist": "uniform", + "time_logit_mean": -1.5, + "time_logit_std": 0.6, + "token_embed_init": "t5_shared", + "token_embed_model_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small", + "token_embed_dim": 512, + "freeze_token_embed": 0, + "freeze_t5_encoder": 1, + "c_min": 1.0, + "c_max": 1024.0, + "c_schedule": "exp", + "concat_self_cond": 0, + "self_cond_ce": 0, + "self_cond_use_cfg_scale": 0, + "self_cond_input_cfg": 0, + "self_cond_cfg_distill": 0, + "self_cond_prob_add": 0, + "self_cond_prob_add_scale": 1.0, + "state_self_cond_prob_add": 1, + "state_self_cond_prob": 0.5, + "state_self_cond_scale": 1.0, + "state_self_cond_normalize": 1, + "state_mask_ratio_min": 0.0, + "state_mask_ratio_max": 0.0, + "self_cond_prob": 0.5, + "self_cond_cfg_min": 0.5, + "self_cond_cfg_max": 5.0, + "self_cond_cfg_formula": "elf", + "self_cond_kl_weight": 0.05, + "self_cond_kl_warmup": 1000, + "self_cond_teacher_temp": 1.0, + "seed": 1234, + "loader_batches_per_rank": 0, + "optimizer_steps_per_epoch": 0, + "steps_per_epoch": 0, + "effective_batch_size": 512 +} +[data] mode=elf_hfds rows=elf_hfds length=1024 vocab=32100 seen=0 dropped=0 bos=1: eos=1:Muon: 58 2D params; Nesterov-AdamW: 78 other paramsMuon: 58 2D params; Nesterov-AdamW: 78 other params + + +[optim] optimizer=muon lr=6.000000e-04 blr=1.000000e-03 effective_batch=512 warmup_steps=200 lr_schedule=constant wd=0.0 loader_batches_per_rank=0 optimizer_steps_per_epoch=0 ema_decay=0.9999 ema_start_steps=0 +[embed] init=t5_shared dim=512 freeze=False freeze_t5_encoder=True model_path=/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small +[arch] time_conditioning=prefix time_tokens=4 use_abs_pos=True state_mask_ratio=[0.0,0.0] time_sample=uniform logit_normal=(-1.5,0.6) +[self_cond] ce=False use_cfg_scale=False input_cfg=False distill=False prob_add=False prob_add_scale=1.0 state_prob_add=True state_prob=0.5 state_scale=1.0 state_norm=True concat=False prob=0.5 scale=[0.5,5.0] formula=elf kl_weight=0.05 kl_warmup=1000 teacher_temp=1.0 +Muon: 58 2D params; Nesterov-AdamW: 78 other params +Muon: 58 2D params; Nesterov-AdamW: 78 other params +t-20260606223951-29b25-worker-0:10452:11953 [2] NCCL INFO NVLS comm 0x9e94a30 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10451:11955 [1] NCCL INFO NVLS comm 0xb3b8290 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10450:11957 [0] NCCL INFO NVLS comm 0xb57f970 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10453:11958 [3] NCCL INFO NVLS comm 0xb306d00 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10456:11960 [6] NCCL INFO NVLS comm 0xa9fcfd0 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10454:11959 [4] NCCL INFO NVLS comm 0xb880ab0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10457:11954 [7] NCCL INFO NVLS comm 0xb194090 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260606223951-29b25-worker-0:10455:11956 [5] NCCL INFO NVLS comm 0xa070080 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +step=50 lr=1.500000e-04 loss=10.2531 ce=10.2531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3801831048331223e-05, 'pos0_bos_top1': 0, 'last_eos_p': 2.6183302907156758e-05, 'last_eos_top1': 0} +step=100 lr=3.000000e-04 loss=9.3265 ce=9.3265 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.073743573622778e-05, 'pos0_bos_top1': 0, 'last_eos_p': 3.387559263501316e-05, 'last_eos_top1': 0} +step=150 lr=4.500000e-04 loss=7.9779 ce=7.9779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1959272493841127e-05, 'pos0_bos_top1': 0, 'last_eos_p': 4.652098141377792e-05, 'last_eos_top1': 0} +step=200 lr=6.000000e-04 loss=6.3874 ce=6.3874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.772459578816779e-05, 'pos0_bos_top1': 0, 'last_eos_p': 4.8116147809196264e-05, 'last_eos_top1': 0} +step=250 lr=6.000000e-04 loss=6.3824 ce=6.3824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.011233588447794e-05, 'pos0_bos_top1': 0, 'last_eos_p': 5.562456135521643e-05, 'last_eos_top1': 0} +step=300 lr=6.000000e-04 loss=5.7973 ce=5.7973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6671560741961e-05, 'pos0_bos_top1': 0, 'last_eos_p': 6.87783322064206e-05, 'last_eos_top1': 0} +step=350 lr=6.000000e-04 loss=5.5618 ce=5.5618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.645047687925398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 9.088359365705401e-05, 'last_eos_top1': 0} +step=400 lr=6.000000e-04 loss=5.6770 ce=5.6770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.299342541955411e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011850986629724503, 'last_eos_top1': 0} +step=450 lr=6.000000e-04 loss=4.6186 ce=4.6186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0735106924548745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011940967669943348, 'last_eos_top1': 0} +step=500 lr=6.000000e-04 loss=4.3524 ce=4.3524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8846108762081712e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001661373389651999, 'last_eos_top1': 0} +step=550 lr=6.000000e-04 loss=4.4254 ce=4.4254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.506689245114103e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000626051623839885, 'last_eos_top1': 0} +step=600 lr=6.000000e-04 loss=4.1125 ce=4.1125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1135884859832004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019440032774582505, 'last_eos_top1': 0} +step=650 lr=6.000000e-04 loss=3.9797 ce=3.9797 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.573487866669893e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006080338731408119, 'last_eos_top1': 0} +step=700 lr=6.000000e-04 loss=4.6736 ce=4.6736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 0.00010387785732746124, 'pos0_bos_top1': 0, 'last_eos_p': 0.008658017963171005, 'last_eos_top1': 0} +step=750 lr=6.000000e-04 loss=4.6547 ce=4.6547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 0.00010251208732370287, 'pos0_bos_top1': 0, 'last_eos_p': 0.005629127845168114, 'last_eos_top1': 0} +step=800 lr=6.000000e-04 loss=4.0736 ce=4.0736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8328885895898566e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003689022734761238, 'last_eos_top1': 0} +step=850 lr=6.000000e-04 loss=4.3321 ce=4.3321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.210050843132194e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002382101956754923, 'last_eos_top1': 0} +step=900 lr=6.000000e-04 loss=3.8422 ce=3.8422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.272755707759643e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016689825570210814, 'last_eos_top1': 0} +step=950 lr=6.000000e-04 loss=3.9030 ce=3.9030 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.722847501805518e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013421443291008472, 'last_eos_top1': 0} +step=1000 lr=6.000000e-04 loss=4.1644 ce=4.1644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.826204531127587e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001350954407826066, 'last_eos_top1': 0} +step=1050 lr=6.000000e-04 loss=4.2269 ce=4.2269 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3703003080299823e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009916236158460379, 'last_eos_top1': 0} +step=1100 lr=6.000000e-04 loss=3.5276 ce=3.5276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.979015451070154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012647062540054321, 'last_eos_top1': 0} +step=1150 lr=6.000000e-04 loss=3.5327 ce=3.5327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3184853742131963e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001528776017948985, 'last_eos_top1': 0} +step=1200 lr=6.000000e-04 loss=4.0325 ce=4.0325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0209007541561732e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015240085776895285, 'last_eos_top1': 0} +step=1250 lr=6.000000e-04 loss=3.7099 ce=3.7099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.173227226710878e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010815944988280535, 'last_eos_top1': 0} +step=1300 lr=6.000000e-04 loss=3.3872 ce=3.3872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9651884031191003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012506050989031792, 'last_eos_top1': 0} +step=1350 lr=6.000000e-04 loss=3.0733 ce=3.0733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.60883189184824e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010790459346026182, 'last_eos_top1': 0} +step=1400 lr=6.000000e-04 loss=3.4694 ce=3.4694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.722086941910675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013917619362473488, 'last_eos_top1': 0} +step=1450 lr=6.000000e-04 loss=3.5099 ce=3.5099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.305904091481352e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017256243154406548, 'last_eos_top1': 0} +step=1500 lr=6.000000e-04 loss=3.3184 ce=3.3184 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.66236701363232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001285388832911849, 'last_eos_top1': 0} +step=1550 lr=6.000000e-04 loss=3.7791 ce=3.7791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.374568096769508e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012384069850668311, 'last_eos_top1': 0} +step=1600 lr=6.000000e-04 loss=3.1608 ce=3.1608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2529644664027728e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001504149753600359, 'last_eos_top1': 0} +step=1650 lr=6.000000e-04 loss=3.6052 ce=3.6052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1577696568565443e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001371248858049512, 'last_eos_top1': 0} +step=1700 lr=6.000000e-04 loss=3.9245 ce=3.9245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.618076208222192e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001606830395758152, 'last_eos_top1': 0} +step=1750 lr=6.000000e-04 loss=3.2002 ce=3.2002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.417329015879659e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017311127157881856, 'last_eos_top1': 0} +step=1800 lr=6.000000e-04 loss=3.2390 ce=3.2390 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.053669438988436e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00220167962834239, 'last_eos_top1': 0} +step=1850 lr=6.000000e-04 loss=3.2885 ce=3.2885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.526034773443826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003042234107851982, 'last_eos_top1': 0} +step=1900 lr=6.000000e-04 loss=3.3326 ce=3.3326 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9677421480300836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020991358906030655, 'last_eos_top1': 0} +step=1950 lr=6.000000e-04 loss=3.6767 ce=3.6767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.774986675533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004171186126768589, 'last_eos_top1': 0} +step=2000 lr=6.000000e-04 loss=2.9856 ce=2.9856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7867071114305872e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00349768646992743, 'last_eos_top1': 0} +step=2050 lr=6.000000e-04 loss=3.3754 ce=3.3754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.530391601700103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004654177464544773, 'last_eos_top1': 0} +step=2100 lr=6.000000e-04 loss=3.4136 ce=3.4136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.907851356008905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003173235571011901, 'last_eos_top1': 0} +step=2150 lr=6.000000e-04 loss=3.1146 ce=3.1146 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0928487174387556e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003830996807664633, 'last_eos_top1': 0} +step=2200 lr=6.000000e-04 loss=3.2364 ce=3.2364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.324247589058359e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004733422305434942, 'last_eos_top1': 0} +step=2250 lr=6.000000e-04 loss=3.1266 ce=3.1266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6857318264083005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043450770899653435, 'last_eos_top1': 0} +step=2300 lr=6.000000e-04 loss=2.7905 ce=2.7905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0708242371038068e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006598710548132658, 'last_eos_top1': 0} +step=2350 lr=6.000000e-04 loss=3.2941 ce=3.2941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.188175575516652e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009538169018924236, 'last_eos_top1': 0} +step=2400 lr=6.000000e-04 loss=3.3082 ce=3.3082 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0487032341188751e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008831417188048363, 'last_eos_top1': 0} +step=2450 lr=6.000000e-04 loss=3.1527 ce=3.1527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2393189687572885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004557142034173012, 'last_eos_top1': 0} +step=2500 lr=6.000000e-04 loss=2.4767 ce=2.4767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.171683031017892e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005824519321322441, 'last_eos_top1': 0} +step=2550 lr=6.000000e-04 loss=3.4327 ce=3.4327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.75948888581479e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008276619017124176, 'last_eos_top1': 0} +step=2600 lr=6.000000e-04 loss=3.3670 ce=3.3670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4341535360435955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010394018143415451, 'last_eos_top1': 0} +step=2650 lr=6.000000e-04 loss=3.3297 ce=3.3297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.880177458398975e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.007535303011536598, 'last_eos_top1': 0} +step=2700 lr=6.000000e-04 loss=2.9162 ce=2.9162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.763013852330914e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.008030209690332413, 'last_eos_top1': 0} +step=2750 lr=6.000000e-04 loss=3.5212 ce=3.5212 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.586084282389493e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.007152436301112175, 'last_eos_top1': 0} +step=2800 lr=6.000000e-04 loss=3.4593 ce=3.4593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.313673222597572e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.013742100447416306, 'last_eos_top1': 0} +step=2850 lr=6.000000e-04 loss=2.8423 ce=2.8423 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.170763420712319e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.00822997372597456, 'last_eos_top1': 0} +step=2900 lr=6.000000e-04 loss=2.9219 ce=2.9219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.232638843859604e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.008281396701931953, 'last_eos_top1': 0} +step=2950 lr=6.000000e-04 loss=2.7458 ce=2.7458 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6922446017561015e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.011427411809563637, 'last_eos_top1': 0} +step=3000 lr=6.000000e-04 loss=2.3841 ce=2.3841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.454329030068038e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01380455307662487, 'last_eos_top1': 0} +step=3050 lr=6.000000e-04 loss=2.5129 ce=2.5129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.889722736654221e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.009778842329978943, 'last_eos_top1': 0} +step=3100 lr=6.000000e-04 loss=3.2394 ce=3.2394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.980688797535549e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.017453504726290703, 'last_eos_top1': 0} +step=3150 lr=6.000000e-04 loss=2.7529 ce=2.7529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.51898403702944e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.009236401878297329, 'last_eos_top1': 0} +step=3200 lr=6.000000e-04 loss=2.8272 ce=2.8272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.583545311717899e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.010657811537384987, 'last_eos_top1': 0} +step=3250 lr=6.000000e-04 loss=3.1988 ce=3.1988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7129709678774816e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01231965608894825, 'last_eos_top1': 0} +step=3300 lr=6.000000e-04 loss=3.0705 ce=3.0705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.973450250465248e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.012690471485257149, 'last_eos_top1': 0} +step=3350 lr=6.000000e-04 loss=2.6421 ce=2.6421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9108898047525145e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.008760485798120499, 'last_eos_top1': 0} +step=3400 lr=6.000000e-04 loss=2.5364 ce=2.5364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.764201205238351e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01053185760974884, 'last_eos_top1': 0} +step=3450 lr=6.000000e-04 loss=3.4805 ce=3.4805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.172222803499608e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.017433661967515945, 'last_eos_top1': 0} +step=3500 lr=6.000000e-04 loss=2.9773 ce=2.9773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.501808004737541e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02168424427509308, 'last_eos_top1': 0} +step=3550 lr=6.000000e-04 loss=2.9124 ce=2.9124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.952117127591919e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.019170239567756653, 'last_eos_top1': 0} +step=3600 lr=6.000000e-04 loss=2.9911 ce=2.9911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.993182068981696e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.024181604385375977, 'last_eos_top1': 0} +step=3650 lr=6.000000e-04 loss=2.9177 ce=2.9177 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.746483680013625e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.021349580958485603, 'last_eos_top1': 0} +step=3700 lr=6.000000e-04 loss=2.8725 ce=2.8725 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.349949899231433e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01658867858350277, 'last_eos_top1': 0} +step=3750 lr=6.000000e-04 loss=2.7113 ce=2.7113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9428271968990884e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.015102908946573734, 'last_eos_top1': 0} +step=3800 lr=6.000000e-04 loss=2.4471 ce=2.4471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.830885356772342e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.010321843437850475, 'last_eos_top1': 0} +step=3850 lr=6.000000e-04 loss=2.9718 ce=2.9718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.210436148037843e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.010996321216225624, 'last_eos_top1': 0} +step=3900 lr=6.000000e-04 loss=2.4899 ce=2.4899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.685914281159057e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.018120426684617996, 'last_eos_top1': 0} +step=3950 lr=6.000000e-04 loss=3.3926 ce=3.3926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.703642606633366e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01992523856461048, 'last_eos_top1': 0} +step=4000 lr=6.000000e-04 loss=2.6989 ce=2.6989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.710908345055941e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.015219107270240784, 'last_eos_top1': 0} +step=4050 lr=6.000000e-04 loss=3.3394 ce=3.3394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.776669465580198e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.013364477083086967, 'last_eos_top1': 0} +step=4100 lr=6.000000e-04 loss=2.9322 ce=2.9322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.20883895408042e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01769949123263359, 'last_eos_top1': 0} +step=4150 lr=6.000000e-04 loss=2.7843 ce=2.7843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.319701585904113e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.018748555332422256, 'last_eos_top1': 0} +step=4200 lr=6.000000e-04 loss=2.7685 ce=2.7685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.607890223771392e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.021574243903160095, 'last_eos_top1': 0} +step=4250 lr=6.000000e-04 loss=2.6629 ce=2.6629 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9598771195414884e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.023443948477506638, 'last_eos_top1': 0} +step=4300 lr=6.000000e-04 loss=2.6317 ce=2.6317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2054118176129123e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01759897917509079, 'last_eos_top1': 0} +step=4350 lr=6.000000e-04 loss=2.5920 ce=2.5920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3117428327168454e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.011566845700144768, 'last_eos_top1': 0} +step=4400 lr=6.000000e-04 loss=2.5581 ce=2.5581 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1407082562727737e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.009198667481541634, 'last_eos_top1': 0} +step=4450 lr=6.000000e-04 loss=2.8984 ce=2.8984 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.250146392019815e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.015449118800461292, 'last_eos_top1': 0} +step=4500 lr=6.000000e-04 loss=2.8425 ce=2.8425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.35962453038519e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01651166006922722, 'last_eos_top1': 0} +step=4550 lr=6.000000e-04 loss=2.5214 ce=2.5214 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.54661107546417e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02051503211259842, 'last_eos_top1': 0} +step=4600 lr=6.000000e-04 loss=2.7612 ce=2.7612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7754733789843158e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02254854515194893, 'last_eos_top1': 0} +step=4650 lr=6.000000e-04 loss=2.8668 ce=2.8668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.169980568098254e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.027828365564346313, 'last_eos_top1': 0} +step=4700 lr=6.000000e-04 loss=3.1003 ce=3.1003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.822204229843919e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.023272652179002762, 'last_eos_top1': 0} +step=4750 lr=6.000000e-04 loss=2.7652 ce=2.7652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2798362781250034e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.016530515626072884, 'last_eos_top1': 0} +step=4800 lr=6.000000e-04 loss=2.8284 ce=2.8284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5430912248557433e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01576833426952362, 'last_eos_top1': 0} +step=4850 lr=6.000000e-04 loss=2.9492 ce=2.9492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.529309644065506e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02426222711801529, 'last_eos_top1': 0} +step=4900 lr=6.000000e-04 loss=3.2491 ce=3.2491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7636653499030217e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.022648457437753677, 'last_eos_top1': 0} +step=4950 lr=6.000000e-04 loss=2.7703 ce=2.7703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3314597064636473e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02541019208729267, 'last_eos_top1': 0} +step=5000 lr=6.000000e-04 loss=2.8934 ce=2.8934 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0414739765328704e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.022253215312957764, 'last_eos_top1': 0} +step=5050 lr=6.000000e-04 loss=2.6487 ce=2.6487 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.88656786753927e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.016012877225875854, 'last_eos_top1': 0} +step=5100 lr=6.000000e-04 loss=2.8791 ce=2.8791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2275138487893855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0280684195458889, 'last_eos_top1': 0} +step=5150 lr=6.000000e-04 loss=2.8224 ce=2.8224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.530652510467917e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02060391753911972, 'last_eos_top1': 0} +step=5200 lr=6.000000e-04 loss=2.5035 ce=2.5035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.305719585318002e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0195072703063488, 'last_eos_top1': 0} +step=5250 lr=6.000000e-04 loss=2.9079 ce=2.9079 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.577098937057599e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.031526118516922, 'last_eos_top1': 0} +step=5300 lr=6.000000e-04 loss=3.0134 ce=3.0134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.058032002125401e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.030827712267637253, 'last_eos_top1': 0} +step=5350 lr=6.000000e-04 loss=2.8968 ce=2.8968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.301925974279584e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02353053167462349, 'last_eos_top1': 0} +step=5400 lr=6.000000e-04 loss=2.6791 ce=2.6791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.345644803535833e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02994992583990097, 'last_eos_top1': 0} +step=5450 lr=6.000000e-04 loss=2.9143 ce=2.9143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0543551525188377e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01446707732975483, 'last_eos_top1': 0} +step=5500 lr=6.000000e-04 loss=2.7043 ce=2.7043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4319069780176505e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.01312106940895319, 'last_eos_top1': 0} +step=5550 lr=6.000000e-04 loss=2.7573 ce=2.7573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8160917509012506e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02174203470349312, 'last_eos_top1': 0} +step=5600 lr=6.000000e-04 loss=2.6672 ce=2.6672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.844394148520223e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.032097600400447845, 'last_eos_top1': 0} +step=5650 lr=6.000000e-04 loss=2.6157 ce=2.6157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6856997692448203e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.018762415274977684, 'last_eos_top1': 0} +step=5700 lr=6.000000e-04 loss=2.2596 ce=2.2596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4977116431255126e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.016046365723013878, 'last_eos_top1': 0} +[elf_hfds direct rank=6 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=100000 rows=91809 buffered=8191 part=openwebtext-t5 +step=5750 lr=6.000000e-04 loss=2.7157 ce=2.7157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.499210604080872e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.023660993203520775, 'last_eos_top1': 0} +step=5800 lr=6.000000e-04 loss=2.8120 ce=2.8120 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.75116552883992e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.02777140960097313, 'last_eos_top1': 0} +step=5850 lr=6.000000e-04 loss=3.3876 ce=3.3876 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7192527366205468e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02962571382522583, 'last_eos_top1': 0} +step=5900 lr=6.000000e-04 loss=2.1066 ce=2.1066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8792886748997262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.03206169605255127, 'last_eos_top1': 0} +step=5950 lr=6.000000e-04 loss=3.5703 ce=3.5703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5131308828131296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04526878148317337, 'last_eos_top1': 0} +step=6000 lr=6.000000e-04 loss=2.4999 ce=2.4999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1047059085030924e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.03876389563083649, 'last_eos_top1': 0} +step=6050 lr=6.000000e-04 loss=2.6230 ce=2.6230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2379864529066253e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04268036037683487, 'last_eos_top1': 0} +step=6100 lr=6.000000e-04 loss=2.9272 ce=2.9272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9907041607657447e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.052099600434303284, 'last_eos_top1': 0} +step=6150 lr=6.000000e-04 loss=2.9715 ce=2.9715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0460489597317064e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04715878516435623, 'last_eos_top1': 0} +step=6200 lr=6.000000e-04 loss=2.9385 ce=2.9385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.462312740040943e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.037037286907434464, 'last_eos_top1': 0} +step=6250 lr=6.000000e-04 loss=2.8325 ce=2.8325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6993932351615513e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04757373780012131, 'last_eos_top1': 0} +step=6300 lr=6.000000e-04 loss=3.1988 ce=3.1988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.668479737621965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.05121888592839241, 'last_eos_top1': 0} +step=6350 lr=6.000000e-04 loss=3.2039 ce=3.2039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2378906123776687e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04772528260946274, 'last_eos_top1': 0} +step=6400 lr=6.000000e-04 loss=2.6912 ce=2.6912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.665316843485925e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.03082674741744995, 'last_eos_top1': 0} +step=6450 lr=6.000000e-04 loss=3.0048 ce=3.0048 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7563626897754148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02196839079260826, 'last_eos_top1': 0} +step=6500 lr=6.000000e-04 loss=2.6043 ce=2.6043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.45429042581236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02808293327689171, 'last_eos_top1': 0} +step=6550 lr=6.000000e-04 loss=2.9940 ce=2.9940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.010666766087525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013752320781350136, 'last_eos_top1': 0} +step=6600 lr=6.000000e-04 loss=3.0881 ce=3.0881 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.265666343097109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.048930566757917404, 'last_eos_top1': 0} +step=6650 lr=6.000000e-04 loss=2.8867 ce=2.8867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6324779507122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.04617920517921448, 'last_eos_top1': 0} +step=6700 lr=6.000000e-04 loss=2.5195 ce=2.5195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.687633918365464e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.025505956262350082, 'last_eos_top1': 0} +step=6750 lr=6.000000e-04 loss=3.4343 ce=3.4343 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1976053378457436e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008850883692502975, 'last_eos_top1': 0} +step=6800 lr=6.000000e-04 loss=3.2396 ce=3.2396 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4377377485507168e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01152629405260086, 'last_eos_top1': 0} +step=6850 lr=6.000000e-04 loss=2.9340 ce=2.9340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.186157745309174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009701713919639587, 'last_eos_top1': 0} +step=6900 lr=6.000000e-04 loss=2.9246 ce=2.9246 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.559830020094523e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008180441334843636, 'last_eos_top1': 0} +step=6950 lr=6.000000e-04 loss=2.3314 ce=2.3314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6501718391737086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007811873219907284, 'last_eos_top1': 0} +step=7000 lr=6.000000e-04 loss=2.7428 ce=2.7428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.334127879701555e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01191839762032032, 'last_eos_top1': 0} +step=7050 lr=6.000000e-04 loss=2.4592 ce=2.4592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.801584054803243e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017090553417801857, 'last_eos_top1': 0} +step=7100 lr=6.000000e-04 loss=3.0556 ce=3.0556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4121280704275705e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01671948842704296, 'last_eos_top1': 0} +step=7150 lr=6.000000e-04 loss=3.0737 ce=3.0737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.908801201963797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010402889922261238, 'last_eos_top1': 0} +step=7200 lr=6.000000e-04 loss=2.7223 ce=2.7223 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0506713806535117e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011276619508862495, 'last_eos_top1': 0} +step=7250 lr=6.000000e-04 loss=3.2675 ce=3.2675 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6301924663130194e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007403482683002949, 'last_eos_top1': 0} +step=7300 lr=6.000000e-04 loss=2.6588 ce=2.6588 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2508280380861834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.020380264148116112, 'last_eos_top1': 0} +step=7350 lr=6.000000e-04 loss=2.4582 ce=2.4582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4770517736906186e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012673798948526382, 'last_eos_top1': 0} +step=7400 lr=6.000000e-04 loss=2.3457 ce=2.3457 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9049535694648512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0160219743847847, 'last_eos_top1': 0} +step=7450 lr=6.000000e-04 loss=2.8728 ce=2.8728 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.96800372377038e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010654217563569546, 'last_eos_top1': 0} +step=7500 lr=6.000000e-04 loss=3.6044 ce=3.6044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.165570913348347e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01976590044796467, 'last_eos_top1': 0} +step=7550 lr=6.000000e-04 loss=2.4489 ce=2.4489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.970139565964928e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.021198518574237823, 'last_eos_top1': 0} +step=7600 lr=6.000000e-04 loss=2.9084 ce=2.9084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.412947990000248e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00814354419708252, 'last_eos_top1': 0} +step=7650 lr=6.000000e-04 loss=2.4152 ce=2.4152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0784733351320028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00654972717165947, 'last_eos_top1': 0} +step=7700 lr=6.000000e-04 loss=2.5714 ce=2.5714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6044603146146983e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005396801047027111, 'last_eos_top1': 0} +step=7750 lr=6.000000e-04 loss=2.6660 ce=2.6660 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.19719014846487e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010935189202427864, 'last_eos_top1': 0} +step=7800 lr=6.000000e-04 loss=2.8543 ce=2.8543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.969232107512653e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009982382878661156, 'last_eos_top1': 0} +step=7850 lr=6.000000e-04 loss=2.7526 ce=2.7526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9888240785803646e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010588088072836399, 'last_eos_top1': 0} +step=7900 lr=6.000000e-04 loss=2.3928 ce=2.3928 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3872559975425247e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010877754539251328, 'last_eos_top1': 0} +step=7950 lr=6.000000e-04 loss=3.1022 ce=3.1022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2155473086750135e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009961912408471107, 'last_eos_top1': 0} +step=8000 lr=6.000000e-04 loss=2.5204 ce=2.5204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1601967344176956e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01243437547236681, 'last_eos_top1': 0} +step=8050 lr=6.000000e-04 loss=2.7181 ce=2.7181 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2259546565474011e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015239731408655643, 'last_eos_top1': 0} +step=8100 lr=6.000000e-04 loss=2.6943 ce=2.6943 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.087611963157542e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010557365603744984, 'last_eos_top1': 0} +step=8150 lr=6.000000e-04 loss=2.7056 ce=2.7056 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.451546374184545e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010349299758672714, 'last_eos_top1': 0} +step=8200 lr=6.000000e-04 loss=2.8887 ce=2.8887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0406867659185082e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01015038788318634, 'last_eos_top1': 0} +step=8250 lr=6.000000e-04 loss=3.0715 ce=3.0715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0971693882311229e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008914860896766186, 'last_eos_top1': 0} +step=8300 lr=6.000000e-04 loss=2.4987 ce=2.4987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.099390364193823e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011219563893973827, 'last_eos_top1': 0} +step=8350 lr=6.000000e-04 loss=2.6400 ce=2.6400 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.203161658253521e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011619050055742264, 'last_eos_top1': 0} +step=8400 lr=6.000000e-04 loss=2.4997 ce=2.4997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.609612789063249e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.025302130728960037, 'last_eos_top1': 0} +step=8450 lr=6.000000e-04 loss=2.9323 ce=2.9323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.691281330655329e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02962186187505722, 'last_eos_top1': 0} +step=8500 lr=6.000000e-04 loss=2.0559 ce=2.0559 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.423592220817227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.025415237993001938, 'last_eos_top1': 0} +step=8550 lr=6.000000e-04 loss=2.7729 ce=2.7729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3568993381340988e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.03044862300157547, 'last_eos_top1': 0} +step=8600 lr=6.000000e-04 loss=2.9995 ce=2.9995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6440571926068515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.027322223410010338, 'last_eos_top1': 0} +step=8650 lr=6.000000e-04 loss=2.6373 ce=2.6373 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.088952538149897e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013708043843507767, 'last_eos_top1': 0} +step=8700 lr=6.000000e-04 loss=2.6554 ce=2.6554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2943364708917215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015362394973635674, 'last_eos_top1': 0} +step=8750 lr=6.000000e-04 loss=2.7849 ce=2.7849 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4354302038555034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01601674035191536, 'last_eos_top1': 0} +step=8800 lr=6.000000e-04 loss=2.7904 ce=2.7904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9459533834597096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017594799399375916, 'last_eos_top1': 0} +step=8850 lr=6.000000e-04 loss=3.1455 ce=3.1455 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4546199458418414e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015399673953652382, 'last_eos_top1': 0} +step=8900 lr=6.000000e-04 loss=2.2194 ce=2.2194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5551278667990118e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010160038247704506, 'last_eos_top1': 0} +step=8950 lr=6.000000e-04 loss=2.3177 ce=2.3177 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5556575565133244e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016516385599970818, 'last_eos_top1': 0} +step=9000 lr=6.000000e-04 loss=2.7490 ce=2.7490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1010924683650956e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032434258610010147, 'last_eos_top1': 0} +step=9050 lr=6.000000e-04 loss=3.0441 ce=3.0441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7230329578742385e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005058696493506432, 'last_eos_top1': 0} +step=9100 lr=6.000000e-04 loss=3.1071 ce=3.1071 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7274182027904317e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054684653878211975, 'last_eos_top1': 0} +step=9150 lr=6.000000e-04 loss=2.5756 ce=2.5756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.959019573405385e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004290000535547733, 'last_eos_top1': 0} +step=9200 lr=6.000000e-04 loss=2.9832 ce=2.9832 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.816511939978227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005010220222175121, 'last_eos_top1': 0} +step=9250 lr=6.000000e-04 loss=3.0600 ce=3.0600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0182899131905288e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005947083234786987, 'last_eos_top1': 0} +step=9300 lr=6.000000e-04 loss=2.7147 ce=2.7147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.01177047326928e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023602177388966084, 'last_eos_top1': 0} +step=9350 lr=6.000000e-04 loss=3.0239 ce=3.0239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.729938028613105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0049450211226940155, 'last_eos_top1': 0} +step=9400 lr=6.000000e-04 loss=2.3724 ce=2.3724 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.305141267948784e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0050139110535383224, 'last_eos_top1': 0} +step=9450 lr=6.000000e-04 loss=2.4610 ce=2.4610 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4486467737006024e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009165993891656399, 'last_eos_top1': 0} +step=9500 lr=6.000000e-04 loss=3.1939 ce=3.1939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4010895938554313e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002168971113860607, 'last_eos_top1': 0} +step=9550 lr=6.000000e-04 loss=2.9346 ce=2.9346 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0346438759588636e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005019490607082844, 'last_eos_top1': 0} +step=9600 lr=6.000000e-04 loss=2.7614 ce=2.7614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1734612598957028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00222994526848197, 'last_eos_top1': 0} +step=9650 lr=6.000000e-04 loss=3.0348 ce=3.0348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6167497960850596e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002010800875723362, 'last_eos_top1': 0} +step=9700 lr=6.000000e-04 loss=2.9535 ce=2.9535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2810157386411447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004503665957599878, 'last_eos_top1': 0} +step=9750 lr=6.000000e-04 loss=2.7829 ce=2.7829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3884397048968822e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007395251654088497, 'last_eos_top1': 0} +step=9800 lr=6.000000e-04 loss=2.6613 ce=2.6613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.645308637118433e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023167498875409365, 'last_eos_top1': 0} +step=9850 lr=6.000000e-04 loss=3.1490 ce=3.1490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0928465599135961e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004759220872074366, 'last_eos_top1': 0} +step=9900 lr=6.000000e-04 loss=3.0169 ce=3.0169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.860632619354874e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010139832273125648, 'last_eos_top1': 0} +step=9950 lr=6.000000e-04 loss=2.9925 ce=2.9925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9468857974279672e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01133008487522602, 'last_eos_top1': 0} +step=10000 lr=6.000000e-04 loss=2.7859 ce=2.7859 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.299050902365707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007525167893618345, 'last_eos_top1': 0} +step=10050 lr=6.000000e-04 loss=2.6207 ce=2.6207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.261618207441643e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004337339662015438, 'last_eos_top1': 0} +step=10100 lr=6.000000e-04 loss=2.8006 ce=2.8006 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2847296349937096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037783156149089336, 'last_eos_top1': 0} +step=10150 lr=6.000000e-04 loss=2.7245 ce=2.7245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.112534573621815e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028823199681937695, 'last_eos_top1': 0} +step=10200 lr=6.000000e-04 loss=2.8358 ce=2.8358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1037642000010237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004045358393341303, 'last_eos_top1': 0} +step=10250 lr=6.000000e-04 loss=2.8291 ce=2.8291 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.033170161070302e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023521138355135918, 'last_eos_top1': 0} +step=10300 lr=6.000000e-04 loss=2.4148 ce=2.4148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1178879503859207e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019727852195501328, 'last_eos_top1': 0} +step=10350 lr=6.000000e-04 loss=3.2542 ce=3.2542 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6103229427244514e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010513747110962868, 'last_eos_top1': 0} +step=10400 lr=6.000000e-04 loss=3.0568 ce=3.0568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3172247892944142e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004672680050134659, 'last_eos_top1': 0} +step=10450 lr=6.000000e-04 loss=2.3386 ce=2.3386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.780943062447477e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003389210905879736, 'last_eos_top1': 0} +step=10500 lr=6.000000e-04 loss=2.8909 ce=2.8909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2264689404401e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002023794688284397, 'last_eos_top1': 0} +step=10550 lr=6.000000e-04 loss=3.0937 ce=3.0937 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9565189733111765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012218320742249489, 'last_eos_top1': 0} +step=10600 lr=6.000000e-04 loss=2.8758 ce=2.8758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.016255009744782e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004249954130500555, 'last_eos_top1': 0} +step=10650 lr=6.000000e-04 loss=2.7501 ce=2.7501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7476954680169e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012448495253920555, 'last_eos_top1': 0} +step=10700 lr=6.000000e-04 loss=2.9599 ce=2.9599 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.180531398858875e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01235809363424778, 'last_eos_top1': 0} +step=10750 lr=6.000000e-04 loss=3.0781 ce=3.0781 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.914493951626355e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018069145735353231, 'last_eos_top1': 0} +step=10800 lr=6.000000e-04 loss=2.5492 ce=2.5492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5800425899215043e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027502593584358692, 'last_eos_top1': 0} +step=10850 lr=6.000000e-04 loss=2.1338 ce=2.1338 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4294682841864415e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00462726503610611, 'last_eos_top1': 0} +step=10900 lr=6.000000e-04 loss=3.1510 ce=3.1510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4085376278671902e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0077679771929979324, 'last_eos_top1': 0} +step=10950 lr=6.000000e-04 loss=2.5235 ce=2.5235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4881416063872166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012763997074216604, 'last_eos_top1': 0} +step=11000 lr=6.000000e-04 loss=2.7496 ce=2.7496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.696743290987797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034090285189449787, 'last_eos_top1': 0} +step=11050 lr=6.000000e-04 loss=2.1778 ce=2.1778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.088905057462398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000667744898237288, 'last_eos_top1': 0} +step=11100 lr=6.000000e-04 loss=3.0333 ce=3.0333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.472172910638619e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004176722140982747, 'last_eos_top1': 0} +step=11150 lr=6.000000e-04 loss=3.1435 ce=3.1435 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4229748558136635e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009677592315711081, 'last_eos_top1': 0} +step=11200 lr=6.000000e-04 loss=2.8882 ce=2.8882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5496332707698457e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009990925900638103, 'last_eos_top1': 0} +step=11250 lr=6.000000e-04 loss=2.9014 ce=2.9014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0828420272446238e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008810576400719583, 'last_eos_top1': 0} +step=11300 lr=6.000000e-04 loss=2.4530 ce=2.4530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.696715844445862e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004988949745893478, 'last_eos_top1': 0} +step=11350 lr=6.000000e-04 loss=2.7594 ce=2.7594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3006627341383137e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028891917318105698, 'last_eos_top1': 0} +step=11400 lr=6.000000e-04 loss=2.8734 ce=2.8734 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3984391443955246e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009456659900024533, 'last_eos_top1': 0} +step=11450 lr=6.000000e-04 loss=2.4974 ce=2.4974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0546833436819725e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001123296096920967, 'last_eos_top1': 0} +step=11500 lr=6.000000e-04 loss=3.2184 ce=3.2184 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.71242661762517e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005092103965580463, 'last_eos_top1': 0} +step=11550 lr=6.000000e-04 loss=2.5430 ce=2.5430 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.433178325300105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013142251409590244, 'last_eos_top1': 0} +step=11600 lr=6.000000e-04 loss=2.4171 ce=2.4171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.78686979477061e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006706690764985979, 'last_eos_top1': 0} +step=11650 lr=6.000000e-04 loss=2.9282 ce=2.9282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.143242666847073e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044449695269577205, 'last_eos_top1': 0} +step=11700 lr=6.000000e-04 loss=2.4041 ce=2.4041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.653998414345551e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023238580615725368, 'last_eos_top1': 0} +step=11750 lr=6.000000e-04 loss=2.3847 ce=2.3847 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.470914624718716e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019352856324985623, 'last_eos_top1': 0} +step=11800 lr=6.000000e-04 loss=2.7178 ce=2.7178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2296603017603047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016744103049859405, 'last_eos_top1': 0} +step=11850 lr=6.000000e-04 loss=3.6526 ce=3.6526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2999232896836475e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010073622688651085, 'last_eos_top1': 0} +step=11900 lr=6.000000e-04 loss=2.4744 ce=2.4744 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1962959433731157e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004702890873886645, 'last_eos_top1': 0} +step=11950 lr=6.000000e-04 loss=3.2912 ce=3.2912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4742628081876319e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007535408367402852, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=200000 rows=191809 buffered=8191 part=openwebtext-t5 +step=12000 lr=6.000000e-04 loss=2.4888 ce=2.4888 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.800338946457487e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047824595822021365, 'last_eos_top1': 0} +step=12050 lr=6.000000e-04 loss=2.7478 ce=2.7478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.625584657944273e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009124744683504105, 'last_eos_top1': 0} +step=12100 lr=6.000000e-04 loss=2.9546 ce=2.9546 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2344571587163955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003375045955181122, 'last_eos_top1': 0} +step=12150 lr=6.000000e-04 loss=2.4937 ce=2.4937 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.498666824772954e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0060505010187625885, 'last_eos_top1': 0} +step=12200 lr=6.000000e-04 loss=2.4193 ce=2.4193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.856324711814523e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020342350471764803, 'last_eos_top1': 0} +step=12250 lr=6.000000e-04 loss=2.7887 ce=2.7887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0265735656721517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010729965288192034, 'last_eos_top1': 0} +step=12300 lr=6.000000e-04 loss=1.9382 ce=1.9382 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9362190869287588e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010735266841948032, 'last_eos_top1': 0} +step=12350 lr=6.000000e-04 loss=2.1446 ce=2.1446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0419774955371395e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005459154490381479, 'last_eos_top1': 0} +step=12400 lr=6.000000e-04 loss=2.5669 ce=2.5669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.948856152012013e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010038978653028607, 'last_eos_top1': 0} +step=12450 lr=6.000000e-04 loss=2.7086 ce=2.7086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6590357922250405e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015047426568344235, 'last_eos_top1': 0} +step=12500 lr=6.000000e-04 loss=2.8087 ce=2.8087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.656533590401523e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024808214511722326, 'last_eos_top1': 0} +step=12550 lr=6.000000e-04 loss=2.7505 ce=2.7505 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.049981387448497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007707376498728991, 'last_eos_top1': 0} +step=12600 lr=6.000000e-04 loss=2.4972 ce=2.4972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7525555449537933e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004082233353983611, 'last_eos_top1': 0} +step=12650 lr=6.000000e-04 loss=2.8208 ce=2.8208 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.093736060895026e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017602643929421902, 'last_eos_top1': 0} +step=12700 lr=6.000000e-04 loss=3.1229 ce=3.1229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.451538446417544e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025390619412064552, 'last_eos_top1': 0} +step=12750 lr=6.000000e-04 loss=2.2997 ce=2.2997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0143394320039079e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014783904189243913, 'last_eos_top1': 0} +step=12800 lr=6.000000e-04 loss=2.5856 ce=2.5856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5036247532407288e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021557433065027, 'last_eos_top1': 0} +step=12850 lr=6.000000e-04 loss=2.4509 ce=2.4509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.852262569940649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004364866763353348, 'last_eos_top1': 0} +step=12900 lr=6.000000e-04 loss=2.7697 ce=2.7697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2468764907680452e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003387105534784496, 'last_eos_top1': 0} +step=12950 lr=6.000000e-04 loss=2.0585 ce=2.0585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.497159352467861e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013177117216400802, 'last_eos_top1': 0} +step=13000 lr=6.000000e-04 loss=3.2463 ce=3.2463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0032355930889025e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031216611387208104, 'last_eos_top1': 0} +step=13050 lr=6.000000e-04 loss=2.4380 ce=2.4380 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1602197446336504e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035624290467239916, 'last_eos_top1': 0} +step=13100 lr=6.000000e-04 loss=2.5969 ce=2.5969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1095051377196796e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003988895914517343, 'last_eos_top1': 0} +step=13150 lr=6.000000e-04 loss=2.9847 ce=2.9847 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0928823030553758e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013194086495786905, 'last_eos_top1': 0} +step=13200 lr=6.000000e-04 loss=2.9851 ce=2.9851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.131433054979425e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007938251947052777, 'last_eos_top1': 0} +step=13250 lr=6.000000e-04 loss=2.7691 ce=2.7691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4850129446131177e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006132209673523903, 'last_eos_top1': 0} +step=13300 lr=6.000000e-04 loss=2.6136 ce=2.6136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.641343260416761e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009456598199903965, 'last_eos_top1': 0} +step=13350 lr=6.000000e-04 loss=2.8139 ce=2.8139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.799543315428309e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004489258280955255, 'last_eos_top1': 0} +step=13400 lr=6.000000e-04 loss=2.7178 ce=2.7178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6827048966661096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010469368426129222, 'last_eos_top1': 0} +step=13450 lr=6.000000e-04 loss=2.4106 ce=2.4106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.942551797488704e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022724554874002934, 'last_eos_top1': 0} +step=13500 lr=6.000000e-04 loss=2.7215 ce=2.7215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3398670489550568e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037880829768255353, 'last_eos_top1': 0} +step=13550 lr=6.000000e-04 loss=2.4730 ce=2.4730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.48130184749607e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005828900029882789, 'last_eos_top1': 0} +step=13600 lr=6.000000e-04 loss=2.6428 ce=2.6428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.480572326632682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002474155626259744, 'last_eos_top1': 0} +step=13650 lr=6.000000e-04 loss=3.4820 ce=3.4820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.395356755703688e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002905346336774528, 'last_eos_top1': 0} +step=13700 lr=6.000000e-04 loss=2.5697 ce=2.5697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9365415937500075e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003251707530580461, 'last_eos_top1': 0} +step=13750 lr=6.000000e-04 loss=3.1257 ce=3.1257 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.815556894754991e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000585615576710552, 'last_eos_top1': 0} +step=13800 lr=6.000000e-04 loss=3.4701 ce=3.4701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2437986697477754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036788510624319315, 'last_eos_top1': 0} +step=13850 lr=6.000000e-04 loss=2.5292 ce=2.5292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5937373468186706e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001178587437607348, 'last_eos_top1': 0} +step=13900 lr=6.000000e-04 loss=2.2517 ce=2.2517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6010973670054227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007577542564831674, 'last_eos_top1': 0} +step=13950 lr=6.000000e-04 loss=2.9940 ce=2.9940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6539139323867857e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006750529864802957, 'last_eos_top1': 0} +step=14000 lr=6.000000e-04 loss=2.3148 ce=2.3148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.608123056939803e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039260098128579557, 'last_eos_top1': 0} +step=14050 lr=6.000000e-04 loss=2.4974 ce=2.4974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3535606740333606e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003723439003806561, 'last_eos_top1': 0} +step=14100 lr=6.000000e-04 loss=2.7506 ce=2.7506 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.295422225666698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005642863688990474, 'last_eos_top1': 0} +step=14150 lr=6.000000e-04 loss=3.0495 ce=3.0495 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.592958844616078e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000748141435906291, 'last_eos_top1': 0} +step=14200 lr=6.000000e-04 loss=2.5166 ce=2.5166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9787473977194168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000754455802962184, 'last_eos_top1': 0} +step=14250 lr=6.000000e-04 loss=2.5439 ce=2.5439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9746290490729734e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004955650074407458, 'last_eos_top1': 0} +step=14300 lr=6.000000e-04 loss=3.1519 ce=3.1519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.718887667986564e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000196247041458264, 'last_eos_top1': 0} +step=14350 lr=6.000000e-04 loss=2.9656 ce=2.9656 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8075246771331877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004007119277957827, 'last_eos_top1': 0} +step=14400 lr=6.000000e-04 loss=2.8250 ce=2.8250 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.218975714640692e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003351816558279097, 'last_eos_top1': 0} +step=14450 lr=6.000000e-04 loss=2.4177 ce=2.4177 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.51208436768502e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007249981281347573, 'last_eos_top1': 0} +step=14500 lr=6.000000e-04 loss=2.6191 ce=2.6191 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.82038082939107e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001360093941912055, 'last_eos_top1': 0} +step=14550 lr=6.000000e-04 loss=2.4390 ce=2.4390 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.290351742180064e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008027014555409551, 'last_eos_top1': 0} +step=14600 lr=6.000000e-04 loss=2.8503 ce=2.8503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.606773705338128e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007045438978821039, 'last_eos_top1': 0} +step=14650 lr=6.000000e-04 loss=3.2375 ce=3.2375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.181866046972573e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009174235165119171, 'last_eos_top1': 0} +step=14700 lr=6.000000e-04 loss=2.6098 ce=2.6098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.394750041887164e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000851195480208844, 'last_eos_top1': 0} +step=14750 lr=6.000000e-04 loss=2.0995 ce=2.0995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.466925409156829e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013294192031025887, 'last_eos_top1': 0} +step=14800 lr=6.000000e-04 loss=2.2764 ce=2.2764 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4085554691264406e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016316587571054697, 'last_eos_top1': 0} +step=14850 lr=6.000000e-04 loss=2.9513 ce=2.9513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4504115824820474e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008554760133847594, 'last_eos_top1': 0} +step=14900 lr=6.000000e-04 loss=2.5945 ce=2.5945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3868426322005689e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008970886701717973, 'last_eos_top1': 0} +step=14950 lr=6.000000e-04 loss=3.0513 ce=3.0513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4449169611907564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00098073435947299, 'last_eos_top1': 0} +step=15000 lr=6.000000e-04 loss=2.9428 ce=2.9428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.455218625778798e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010677773971110582, 'last_eos_top1': 0} +step=15050 lr=6.000000e-04 loss=2.6189 ce=2.6189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5545250436116476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007823894266039133, 'last_eos_top1': 0} +step=15100 lr=6.000000e-04 loss=2.8495 ce=2.8495 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7625228540273383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001379616791382432, 'last_eos_top1': 0} +step=15150 lr=6.000000e-04 loss=3.0021 ce=3.0021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.686441075638868e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010611184407025576, 'last_eos_top1': 0} +step=15200 lr=6.000000e-04 loss=2.6517 ce=2.6517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2903934475616552e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009403664153069258, 'last_eos_top1': 0} +step=15250 lr=6.000000e-04 loss=2.1079 ce=2.1079 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7128082618000917e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017772185383364558, 'last_eos_top1': 0} +step=15300 lr=6.000000e-04 loss=2.6804 ce=2.6804 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5835899577941746e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014531335327774286, 'last_eos_top1': 0} +step=15350 lr=6.000000e-04 loss=3.5047 ce=3.5047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5415700292796828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010851696133613586, 'last_eos_top1': 0} +step=15400 lr=6.000000e-04 loss=2.4564 ce=2.4564 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.685445360839367e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010101315565407276, 'last_eos_top1': 0} +step=15450 lr=6.000000e-04 loss=2.6222 ce=2.6222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2418294217204675e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012554751010611653, 'last_eos_top1': 0} +step=15500 lr=6.000000e-04 loss=2.6058 ce=2.6058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 0.00019673258066177368, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032289973460137844, 'last_eos_top1': 0} +step=15550 lr=6.000000e-04 loss=3.2754 ce=3.2754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6538568161195144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004510238068178296, 'last_eos_top1': 0} +step=15600 lr=6.000000e-04 loss=2.4053 ce=2.4053 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4133787266910076e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000877709942869842, 'last_eos_top1': 0} +step=15650 lr=6.000000e-04 loss=2.8011 ce=2.8011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7532231140648946e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011860914528369904, 'last_eos_top1': 0} +step=15700 lr=6.000000e-04 loss=2.4995 ce=2.4995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9480564990080893e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018000784330070019, 'last_eos_top1': 0} +step=15750 lr=6.000000e-04 loss=2.1675 ce=2.1675 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3610420865006745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016709278570488095, 'last_eos_top1': 0} +step=15800 lr=6.000000e-04 loss=2.4858 ce=2.4858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.09933748212643e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007711363141424954, 'last_eos_top1': 0} +step=15850 lr=6.000000e-04 loss=2.6374 ce=2.6374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7082933229394257e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009728852892294526, 'last_eos_top1': 0} +step=15900 lr=6.000000e-04 loss=3.0563 ce=3.0563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3388943443715107e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005409744335338473, 'last_eos_top1': 0} +step=15950 lr=6.000000e-04 loss=2.8827 ce=2.8827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6126468835864216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003424074384383857, 'last_eos_top1': 0} +step=16000 lr=6.000000e-04 loss=2.4983 ce=2.4983 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2723366782884113e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007428853423334658, 'last_eos_top1': 0} +step=16050 lr=6.000000e-04 loss=2.6331 ce=2.6331 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.01058030850254e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002635395503602922, 'last_eos_top1': 0} +step=16100 lr=6.000000e-04 loss=2.5127 ce=2.5127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7287729860981926e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005289375549182296, 'last_eos_top1': 0} +step=16150 lr=6.000000e-04 loss=2.4433 ce=2.4433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.599563559779199e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006246366538107395, 'last_eos_top1': 0} +step=16200 lr=6.000000e-04 loss=2.5456 ce=2.5456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6098953134787735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038966850843280554, 'last_eos_top1': 0} +step=16250 lr=6.000000e-04 loss=2.9638 ce=2.9638 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.79953279561596e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040211217128671706, 'last_eos_top1': 0} +step=16300 lr=6.000000e-04 loss=3.0992 ce=3.0992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6020700059016235e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008097285171970725, 'last_eos_top1': 0} +step=16350 lr=6.000000e-04 loss=2.7322 ce=2.7322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7350354028167203e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007567626889795065, 'last_eos_top1': 0} +step=16400 lr=6.000000e-04 loss=2.5609 ce=2.5609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4884208212606609e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000698694318998605, 'last_eos_top1': 0} +step=16450 lr=6.000000e-04 loss=2.6842 ce=2.6842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.76172106876038e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001095354906283319, 'last_eos_top1': 0} +step=16500 lr=6.000000e-04 loss=2.8586 ce=2.8586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2271886589587666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012009002966806293, 'last_eos_top1': 0} +step=16550 lr=6.000000e-04 loss=2.4170 ce=2.4170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.059842341230251e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006406009197235107, 'last_eos_top1': 0} +step=16600 lr=6.000000e-04 loss=2.1656 ce=2.1656 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.747238613665104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015582471387460828, 'last_eos_top1': 0} +step=16650 lr=6.000000e-04 loss=2.7362 ce=2.7362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4197829841577914e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042131543159484863, 'last_eos_top1': 0} +step=16700 lr=6.000000e-04 loss=3.0055 ce=3.0055 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0966472220607102e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009188856347464025, 'last_eos_top1': 0} +step=16750 lr=6.000000e-04 loss=2.8219 ce=2.8219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.544148047600174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00025531958090141416, 'last_eos_top1': 0} +step=16800 lr=6.000000e-04 loss=2.6742 ce=2.6742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4932864360162057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002805591793730855, 'last_eos_top1': 0} +step=16850 lr=6.000000e-04 loss=2.5721 ce=2.5721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4478223824407905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006714261835440993, 'last_eos_top1': 0} +step=16900 lr=6.000000e-04 loss=2.9084 ce=2.9084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.332136763376184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004265965544618666, 'last_eos_top1': 0} +step=16950 lr=6.000000e-04 loss=3.1313 ce=3.1313 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.025681169470772e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006705132545903325, 'last_eos_top1': 0} +step=17000 lr=6.000000e-04 loss=3.1455 ce=3.1455 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.929413883772213e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005253100534901023, 'last_eos_top1': 0} +step=17050 lr=6.000000e-04 loss=2.9137 ce=2.9137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.53980654431507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004374299314804375, 'last_eos_top1': 0} +step=17100 lr=6.000000e-04 loss=2.4239 ce=2.4239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6375130144297145e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00041549844900146127, 'last_eos_top1': 0} +step=17150 lr=6.000000e-04 loss=2.4976 ce=2.4976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1305587476235814e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018367930897511542, 'last_eos_top1': 0} +step=17200 lr=6.000000e-04 loss=3.1040 ce=3.1040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3691125786863267e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000400543212890625, 'last_eos_top1': 0} +step=17250 lr=6.000000e-04 loss=2.5439 ce=2.5439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.117902325466275e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015485379844903946, 'last_eos_top1': 0} +step=17300 lr=6.000000e-04 loss=2.7381 ce=2.7381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4993758415803313e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038009427953511477, 'last_eos_top1': 0} +step=17350 lr=6.000000e-04 loss=2.7221 ce=2.7221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.61865827976726e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028654432389885187, 'last_eos_top1': 0} +step=17400 lr=6.000000e-04 loss=2.7589 ce=2.7589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.932682062033564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003451227385085076, 'last_eos_top1': 0} +step=17450 lr=6.000000e-04 loss=2.4795 ce=2.4795 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6099982531159185e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027505873003974557, 'last_eos_top1': 0} +step=17500 lr=6.000000e-04 loss=2.2771 ce=2.2771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.704541475395672e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000538378139026463, 'last_eos_top1': 0} +step=17550 lr=6.000000e-04 loss=2.4882 ce=2.4882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4368395568453707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002415408962406218, 'last_eos_top1': 0} +step=17600 lr=6.000000e-04 loss=2.8230 ce=2.8230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9862574845319614e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005609591025859118, 'last_eos_top1': 0} +step=17650 lr=6.000000e-04 loss=2.7500 ce=2.7500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2102089385734871e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039451560587622225, 'last_eos_top1': 0} +step=17700 lr=6.000000e-04 loss=2.7402 ce=2.7402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.80018973373808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001077552791684866, 'last_eos_top1': 0} +step=17750 lr=6.000000e-04 loss=2.7936 ce=2.7936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0990609553118702e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003800542326644063, 'last_eos_top1': 0} +step=17800 lr=6.000000e-04 loss=2.4627 ce=2.4627 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.597848313394934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044833807623945177, 'last_eos_top1': 0} +step=17850 lr=6.000000e-04 loss=2.9526 ce=2.9526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7605608920566738e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000918503908906132, 'last_eos_top1': 0} +step=17900 lr=6.000000e-04 loss=2.8225 ce=2.8225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.606858055922203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003305164282210171, 'last_eos_top1': 0} +step=17950 lr=6.000000e-04 loss=2.4489 ce=2.4489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.190306754026096e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017080767429433763, 'last_eos_top1': 0} +step=18000 lr=6.000000e-04 loss=2.6119 ce=2.6119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3456752640195191e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002243131457362324, 'last_eos_top1': 0} +step=18050 lr=6.000000e-04 loss=2.6448 ce=2.6448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.638444235955831e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001580765238031745, 'last_eos_top1': 0} +step=18100 lr=6.000000e-04 loss=2.5874 ce=2.5874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.246101980243111e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014394446043297648, 'last_eos_top1': 0} +step=18150 lr=6.000000e-04 loss=2.7003 ce=2.7003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0965729416057002e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003469470248091966, 'last_eos_top1': 0} +step=18200 lr=6.000000e-04 loss=3.0578 ce=3.0578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2048957614751998e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004520227084867656, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=300000 rows=291809 buffered=8191 part=openwebtext-t5 +step=18250 lr=6.000000e-04 loss=1.9912 ce=1.9912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.591640496684704e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005989507772028446, 'last_eos_top1': 0} +step=18300 lr=6.000000e-04 loss=2.7243 ce=2.7243 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1616228221100755e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040815924876369536, 'last_eos_top1': 0} +step=18350 lr=6.000000e-04 loss=2.1141 ce=2.1141 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2212701626122e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006950331735424697, 'last_eos_top1': 0} +step=18400 lr=6.000000e-04 loss=2.1448 ce=2.1448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.328947786940262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006198357441462576, 'last_eos_top1': 0} +step=18450 lr=6.000000e-04 loss=2.8529 ce=2.8529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9725395506829955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004433642025105655, 'last_eos_top1': 0} +step=18500 lr=6.000000e-04 loss=2.7607 ce=2.7607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6185415006475523e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000318837002851069, 'last_eos_top1': 0} +step=18550 lr=6.000000e-04 loss=2.7178 ce=2.7178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2973676095716655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038193119689822197, 'last_eos_top1': 0} +step=18600 lr=6.000000e-04 loss=3.2245 ce=3.2245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1015892596333288e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035042467061430216, 'last_eos_top1': 0} +step=18650 lr=6.000000e-04 loss=2.5676 ce=2.5676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4937880223442335e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000754139618948102, 'last_eos_top1': 0} +step=18700 lr=6.000000e-04 loss=2.2436 ce=2.2436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.556109757279046e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004948140121996403, 'last_eos_top1': 0} +step=18750 lr=6.000000e-04 loss=2.7382 ce=2.7382 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4437907995888963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007511784788221121, 'last_eos_top1': 0} +step=18800 lr=6.000000e-04 loss=2.6104 ce=2.6104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.481185897020623e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009302757098339498, 'last_eos_top1': 0} +step=18850 lr=6.000000e-04 loss=2.4260 ce=2.4260 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5859030099818483e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004813050327356905, 'last_eos_top1': 0} +step=18900 lr=6.000000e-04 loss=2.3830 ce=2.3830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.863604277081322e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007530984003096819, 'last_eos_top1': 0} +step=18950 lr=6.000000e-04 loss=2.9805 ce=2.9805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1922134945052676e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042482814751565456, 'last_eos_top1': 0} +step=19000 lr=6.000000e-04 loss=2.4119 ce=2.4119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7590213246876374e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005108428886160254, 'last_eos_top1': 0} +step=19050 lr=6.000000e-04 loss=2.8232 ce=2.8232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.374598003982101e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002603446482680738, 'last_eos_top1': 0} +step=19100 lr=6.000000e-04 loss=3.1350 ce=3.1350 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0971248886780813e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006369848269969225, 'last_eos_top1': 0} +step=19150 lr=6.000000e-04 loss=2.6302 ce=2.6302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.333134034415707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006381960120052099, 'last_eos_top1': 0} +step=19200 lr=6.000000e-04 loss=3.0057 ce=3.0057 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.722577836626442e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032672873931005597, 'last_eos_top1': 0} +step=19250 lr=6.000000e-04 loss=2.7974 ce=2.7974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.009564013860654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007432224228978157, 'last_eos_top1': 0} +step=19300 lr=6.000000e-04 loss=3.0424 ce=3.0424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.483311492251232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046886084601283073, 'last_eos_top1': 0} +step=19350 lr=6.000000e-04 loss=2.3882 ce=2.3882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.079798261751421e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018441517022438347, 'last_eos_top1': 0} +step=19400 lr=6.000000e-04 loss=1.9665 ce=1.9665 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0190289685851894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003577677416615188, 'last_eos_top1': 0} +step=19450 lr=6.000000e-04 loss=2.5495 ce=2.5495 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.618997864890844e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002598505816422403, 'last_eos_top1': 0} +step=19500 lr=6.000000e-04 loss=2.7877 ce=2.7877 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4796252546366304e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000641670310869813, 'last_eos_top1': 0} +step=19550 lr=6.000000e-04 loss=3.0398 ce=3.0398 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0059513442683965e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006586129311472178, 'last_eos_top1': 0} +step=19600 lr=6.000000e-04 loss=2.4754 ce=2.4754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.189025382103864e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004690024652518332, 'last_eos_top1': 0} +step=19650 lr=6.000000e-04 loss=3.3262 ce=3.3262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.978686648537405e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006737797521054745, 'last_eos_top1': 0} +step=19700 lr=6.000000e-04 loss=2.7026 ce=2.7026 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.606536898179911e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009449140634387732, 'last_eos_top1': 0} +step=19750 lr=6.000000e-04 loss=2.7675 ce=2.7675 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2222901204950176e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007958154892548919, 'last_eos_top1': 0} +step=19800 lr=6.000000e-04 loss=2.0325 ce=2.0325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.257742587768007e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046249284059740603, 'last_eos_top1': 0} +step=19850 lr=6.000000e-04 loss=2.2609 ce=2.2609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4551358617609367e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00117168587166816, 'last_eos_top1': 0} +step=19900 lr=6.000000e-04 loss=2.3990 ce=2.3990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.287658809218556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016765461768954992, 'last_eos_top1': 0} +step=19950 lr=6.000000e-04 loss=2.6271 ce=2.6271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.383809260209091e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004723680904135108, 'last_eos_top1': 0} +step=20000 lr=6.000000e-04 loss=2.1229 ce=2.1229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.052765037456993e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003267546999268234, 'last_eos_top1': 0} +step=20050 lr=6.000000e-04 loss=2.4290 ce=2.4290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.288940080092289e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020012017921544611, 'last_eos_top1': 0} +step=20100 lr=6.000000e-04 loss=2.6995 ce=2.6995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.574867140647257e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003362590214237571, 'last_eos_top1': 0} +step=20150 lr=6.000000e-04 loss=2.7902 ce=2.7902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.865976269589737e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003694802289828658, 'last_eos_top1': 0} +step=20200 lr=6.000000e-04 loss=2.7327 ce=2.7327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.970255991385784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003648136043921113, 'last_eos_top1': 0} +step=20250 lr=6.000000e-04 loss=2.5085 ce=2.5085 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1710248145391233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005583569873124361, 'last_eos_top1': 0} +step=20300 lr=6.000000e-04 loss=2.4778 ce=2.4778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0309497459675185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003422548761591315, 'last_eos_top1': 0} +step=20350 lr=6.000000e-04 loss=2.1668 ce=2.1668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2579249414557125e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029937364161014557, 'last_eos_top1': 0} +step=20400 lr=6.000000e-04 loss=3.2848 ce=3.2848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3302674233273137e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004008596297353506, 'last_eos_top1': 0} +step=20450 lr=6.000000e-04 loss=2.3899 ce=2.3899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5153833373915404e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004651751951314509, 'last_eos_top1': 0} +step=20500 lr=6.000000e-04 loss=2.4378 ce=2.4378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6844434867380187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005075991502963006, 'last_eos_top1': 0} +step=20550 lr=6.000000e-04 loss=2.6931 ce=2.6931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.785080818692222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007041705539450049, 'last_eos_top1': 0} +step=20600 lr=6.000000e-04 loss=2.4732 ce=2.4732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7725167456083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039366917917504907, 'last_eos_top1': 0} +step=20650 lr=6.000000e-04 loss=2.4638 ce=2.4638 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.273309994256124e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000625294866040349, 'last_eos_top1': 0} +step=20700 lr=6.000000e-04 loss=2.4718 ce=2.4718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.659489059122279e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002113706897944212, 'last_eos_top1': 0} +step=20750 lr=6.000000e-04 loss=2.8730 ce=2.8730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.189827111782506e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001449516974389553, 'last_eos_top1': 0} +step=20800 lr=6.000000e-04 loss=3.1160 ce=3.1160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3460353986592963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010150960879400373, 'last_eos_top1': 0} +step=20850 lr=6.000000e-04 loss=3.0126 ce=3.0126 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.960692866821773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000582035630941391, 'last_eos_top1': 0} +step=20900 lr=6.000000e-04 loss=2.5516 ce=2.5516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3836373909725808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005306355888023973, 'last_eos_top1': 0} +step=20950 lr=6.000000e-04 loss=2.4908 ce=2.4908 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4481219952576794e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000634564203210175, 'last_eos_top1': 0} +step=21000 lr=6.000000e-04 loss=2.5280 ce=2.5280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.249865353107452e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008417006465606391, 'last_eos_top1': 0} +step=21050 lr=6.000000e-04 loss=2.4596 ce=2.4596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3034673833753914e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011762899812310934, 'last_eos_top1': 0} +step=21100 lr=6.000000e-04 loss=2.4853 ce=2.4853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1578506322111934e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002484132710378617, 'last_eos_top1': 0} +step=21150 lr=6.000000e-04 loss=3.0199 ce=3.0199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1217025530640967e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000614746066275984, 'last_eos_top1': 0} +step=21200 lr=6.000000e-04 loss=2.6903 ce=2.6903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5605301086907275e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011377506889402866, 'last_eos_top1': 0} +step=21250 lr=6.000000e-04 loss=2.5140 ce=2.5140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.982015060610138e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010665771551430225, 'last_eos_top1': 0} +step=21300 lr=6.000000e-04 loss=2.4058 ce=2.4058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.904727832879871e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002272783312946558, 'last_eos_top1': 0} +step=21350 lr=6.000000e-04 loss=2.7100 ce=2.7100 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4531405870220624e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005253132549114525, 'last_eos_top1': 0} +step=21400 lr=6.000000e-04 loss=1.9597 ce=1.9597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.262741514248773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044789339881390333, 'last_eos_top1': 0} +step=21450 lr=6.000000e-04 loss=2.6747 ce=2.6747 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9248402168159373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003315770882181823, 'last_eos_top1': 0} +step=21500 lr=6.000000e-04 loss=2.8531 ce=2.8531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3504068192560226e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001455660560168326, 'last_eos_top1': 0} +step=21550 lr=6.000000e-04 loss=3.0187 ce=3.0187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.761773273756262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004769992083311081, 'last_eos_top1': 0} +step=21600 lr=6.000000e-04 loss=2.8824 ce=2.8824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1456411812105216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001054686727002263, 'last_eos_top1': 0} +step=21650 lr=6.000000e-04 loss=2.5146 ce=2.5146 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6556286936975084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007643569260835648, 'last_eos_top1': 0} +step=21700 lr=6.000000e-04 loss=2.2987 ce=2.2987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2509918633440975e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016278550028800964, 'last_eos_top1': 0} +step=21750 lr=6.000000e-04 loss=2.6094 ce=2.6094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2016301954863593e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010027644457295537, 'last_eos_top1': 0} +step=21800 lr=6.000000e-04 loss=3.0191 ce=3.0191 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5651752619305626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021090141963213682, 'last_eos_top1': 0} +step=21850 lr=6.000000e-04 loss=2.6682 ce=2.6682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6809988159802742e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000301157939247787, 'last_eos_top1': 0} +step=21900 lr=6.000000e-04 loss=2.6113 ce=2.6113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0845618564635515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005359961651265621, 'last_eos_top1': 0} +step=21950 lr=6.000000e-04 loss=2.6879 ce=2.6879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5255523976520635e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006380566628649831, 'last_eos_top1': 0} +step=22000 lr=6.000000e-04 loss=2.9548 ce=2.9548 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7193309759022668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008333909208886325, 'last_eos_top1': 0} +step=22050 lr=6.000000e-04 loss=2.7109 ce=2.7109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5777721273479983e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003828934859484434, 'last_eos_top1': 0} +step=22100 lr=6.000000e-04 loss=2.2836 ce=2.2836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6915697415242903e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002828456345014274, 'last_eos_top1': 0} +step=22150 lr=6.000000e-04 loss=2.6843 ce=2.6843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2302418326726183e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00048286031233146787, 'last_eos_top1': 0} +step=22200 lr=6.000000e-04 loss=2.5685 ce=2.5685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.963520480785519e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007657420355826616, 'last_eos_top1': 0} +step=22250 lr=6.000000e-04 loss=2.2086 ce=2.2086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7181380826514214e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005473907804116607, 'last_eos_top1': 0} +step=22300 lr=6.000000e-04 loss=3.1447 ce=3.1447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.682962778839283e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032288694637827575, 'last_eos_top1': 0} +step=22350 lr=6.000000e-04 loss=2.5882 ce=2.5882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.704731195990462e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001075958600267768, 'last_eos_top1': 0} +step=22400 lr=6.000000e-04 loss=2.3945 ce=2.3945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4426899724639952e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006633307202719152, 'last_eos_top1': 0} +step=22450 lr=6.000000e-04 loss=2.5285 ce=2.5285 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.258984699845314e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009566907538101077, 'last_eos_top1': 0} +step=22500 lr=6.000000e-04 loss=2.5240 ce=2.5240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.199200596602168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005749279516749084, 'last_eos_top1': 0} +step=22550 lr=6.000000e-04 loss=2.4513 ce=2.4513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9118913516867906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017208348726853728, 'last_eos_top1': 0} +step=22600 lr=6.000000e-04 loss=2.8376 ce=2.8376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.584358975989744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00485952477902174, 'last_eos_top1': 0} +step=22650 lr=6.000000e-04 loss=2.4011 ce=2.4011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3350627341424115e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020390113350003958, 'last_eos_top1': 0} +step=22700 lr=6.000000e-04 loss=2.6692 ce=2.6692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7396261682733893e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038556521758437157, 'last_eos_top1': 0} +step=22750 lr=6.000000e-04 loss=2.6688 ce=2.6688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9965601647272706e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005755445919930935, 'last_eos_top1': 0} +step=22800 lr=6.000000e-04 loss=3.3491 ce=3.3491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.637550096551422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004177922382950783, 'last_eos_top1': 0} +step=22850 lr=6.000000e-04 loss=2.7424 ce=2.7424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4128490875009447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009984304197132587, 'last_eos_top1': 0} +step=22900 lr=6.000000e-04 loss=2.5907 ce=2.5907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.727195507963188e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013011354021728039, 'last_eos_top1': 0} +step=22950 lr=6.000000e-04 loss=3.0080 ce=3.0080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6995610824087635e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017120586708188057, 'last_eos_top1': 0} +step=23000 lr=6.000000e-04 loss=2.8005 ce=2.8005 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.107026375597343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010576462373137474, 'last_eos_top1': 0} +step=23050 lr=6.000000e-04 loss=2.2832 ce=2.2832 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2393104043439962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00192415586207062, 'last_eos_top1': 0} +step=23100 lr=6.000000e-04 loss=2.0741 ce=2.0741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.514742638799362e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027129044756293297, 'last_eos_top1': 0} +step=23150 lr=6.000000e-04 loss=2.4746 ce=2.4746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9036815501749516e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022987639531493187, 'last_eos_top1': 0} +step=23200 lr=6.000000e-04 loss=3.1680 ce=3.1680 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2363188034214545e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013042471837252378, 'last_eos_top1': 0} +step=23250 lr=6.000000e-04 loss=2.3641 ce=2.3641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.406146662309766e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011261510662734509, 'last_eos_top1': 0} +step=23300 lr=6.000000e-04 loss=2.6509 ce=2.6509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7190101061714813e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025916823651641607, 'last_eos_top1': 0} +step=23350 lr=6.000000e-04 loss=2.6721 ce=2.6721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.741988307912834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004359006881713867, 'last_eos_top1': 0} +step=23400 lr=6.000000e-04 loss=2.5945 ce=2.5945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.473977135901805e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019155878107994795, 'last_eos_top1': 0} +step=23450 lr=6.000000e-04 loss=2.6858 ce=2.6858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7154284907737747e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002777093555778265, 'last_eos_top1': 0} +step=23500 lr=6.000000e-04 loss=2.6357 ce=2.6357 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.344245826359838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004592364653944969, 'last_eos_top1': 0} +step=23550 lr=6.000000e-04 loss=2.0032 ce=2.0032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.56745915976353e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005123870447278023, 'last_eos_top1': 0} +step=23600 lr=6.000000e-04 loss=3.0475 ce=3.0475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8500453481683508e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021111070236656815, 'last_eos_top1': 0} +step=23650 lr=6.000000e-04 loss=2.6001 ce=2.6001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9650247850222513e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007823857013136148, 'last_eos_top1': 0} +step=23700 lr=6.000000e-04 loss=2.8138 ce=2.8138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5761581127881072e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005506898742169142, 'last_eos_top1': 0} +step=23750 lr=6.000000e-04 loss=2.5778 ce=2.5778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4665441742399707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003508809022605419, 'last_eos_top1': 0} +step=23800 lr=6.000000e-04 loss=2.5554 ce=2.5554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8935564841958694e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011739848414435983, 'last_eos_top1': 0} +step=23850 lr=6.000000e-04 loss=2.7576 ce=2.7576 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.368450026959181e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004613497760146856, 'last_eos_top1': 0} +step=23900 lr=6.000000e-04 loss=2.5503 ce=2.5503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3603580959606916e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004265752155333757, 'last_eos_top1': 0} +step=23950 lr=6.000000e-04 loss=3.6022 ce=3.6022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.632496686303057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009407860925421119, 'last_eos_top1': 0} +step=24000 lr=6.000000e-04 loss=2.6898 ce=2.6898 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6634250136557966e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028231411706656218, 'last_eos_top1': 0} +step=24050 lr=6.000000e-04 loss=2.5184 ce=2.5184 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.362107236171141e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002104594372212887, 'last_eos_top1': 0} +step=24100 lr=6.000000e-04 loss=2.6664 ce=2.6664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.56828240910545e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004953361349180341, 'last_eos_top1': 0} +step=24150 lr=6.000000e-04 loss=2.7762 ce=2.7762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7920259526581503e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009798454120755196, 'last_eos_top1': 0} +step=24200 lr=6.000000e-04 loss=1.9153 ce=1.9153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.875688707921654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010790596716105938, 'last_eos_top1': 0} +step=24250 lr=6.000000e-04 loss=2.2829 ce=2.2829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.637909296434373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001586263533681631, 'last_eos_top1': 0} +step=24300 lr=6.000000e-04 loss=2.0159 ce=2.0159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5556299078743905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027896068058907986, 'last_eos_top1': 0} +step=24350 lr=6.000000e-04 loss=2.5310 ce=2.5310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7535068764118478e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004246038850396872, 'last_eos_top1': 0} +step=24400 lr=6.000000e-04 loss=1.7176 ce=1.7176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.365071850363165e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006191389402374625, 'last_eos_top1': 0} +step=24450 lr=6.000000e-04 loss=2.1957 ce=2.1957 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4152504415251315e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016295217210426927, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=3] docs=400000 rows=391809 buffered=8191 part=openwebtext-t5 + +step=24500 lr=6.000000e-04 loss=2.9890 ce=2.9890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.620557592716068e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004752547247335315, 'last_eos_top1': 0} +step=24550 lr=6.000000e-04 loss=2.4104 ce=2.4104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.639111284632236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005822516977787018, 'last_eos_top1': 0} +step=24600 lr=6.000000e-04 loss=2.4828 ce=2.4828 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.610828182194382e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019126124680042267, 'last_eos_top1': 0} +step=24650 lr=6.000000e-04 loss=2.6535 ce=2.6535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9533380661159754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007612188346683979, 'last_eos_top1': 0} +step=24700 lr=6.000000e-04 loss=2.6263 ce=2.6263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6221382692456245e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039060208946466446, 'last_eos_top1': 0} +step=24750 lr=6.000000e-04 loss=2.0210 ce=2.0210 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.665596108883619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0045173279941082, 'last_eos_top1': 0} +step=24800 lr=6.000000e-04 loss=2.5705 ce=2.5705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.904428977984935e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004826704040169716, 'last_eos_top1': 0} +step=24850 lr=6.000000e-04 loss=2.4078 ce=2.4078 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.63637406937778e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018407276365906, 'last_eos_top1': 0} +step=24900 lr=6.000000e-04 loss=3.0118 ce=3.0118 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.949556462932378e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005219237646088004, 'last_eos_top1': 0} +step=24950 lr=6.000000e-04 loss=2.7907 ce=2.7907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3299576645949855e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001797929173335433, 'last_eos_top1': 0} +step=25000 lr=6.000000e-04 loss=2.4029 ce=2.4029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.842976053012535e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024215485900640488, 'last_eos_top1': 0} +step=25050 lr=6.000000e-04 loss=2.6264 ce=2.6264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8277605451876298e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004941681399941444, 'last_eos_top1': 0} +step=25100 lr=6.000000e-04 loss=2.9635 ce=2.9635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7171172405360267e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004795424756594002, 'last_eos_top1': 0} +step=25150 lr=6.000000e-04 loss=2.2582 ce=2.2582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.294974158052355e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008418129291385412, 'last_eos_top1': 0} +step=25200 lr=6.000000e-04 loss=2.6494 ce=2.6494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1253716113278642e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005859573138877749, 'last_eos_top1': 0} +step=25250 lr=6.000000e-04 loss=2.5766 ce=2.5766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1277595553547144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0040796417742967606, 'last_eos_top1': 0} +step=25300 lr=6.000000e-04 loss=2.2935 ce=2.2935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.775827695382759e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006590555887669325, 'last_eos_top1': 0} +step=25350 lr=6.000000e-04 loss=2.9205 ce=2.9205 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6300447896355763e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022210380993783474, 'last_eos_top1': 0} +step=25400 lr=6.000000e-04 loss=2.5878 ce=2.5878 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0398394806543365e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036657629534602165, 'last_eos_top1': 0} +step=25450 lr=6.000000e-04 loss=2.9935 ce=2.9935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.493165296385996e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003814496798440814, 'last_eos_top1': 0} +step=25500 lr=6.000000e-04 loss=2.4598 ce=2.4598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0172381482552737e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005167917697690427, 'last_eos_top1': 0} +step=25550 lr=6.000000e-04 loss=2.9890 ce=2.9890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.618475653231144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017034669872373343, 'last_eos_top1': 0} +step=25600 lr=6.000000e-04 loss=2.1453 ce=2.1453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6387455616495572e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023053197655826807, 'last_eos_top1': 0} +step=25650 lr=6.000000e-04 loss=2.7460 ce=2.7460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1959935363847762e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058204480446875095, 'last_eos_top1': 0} +step=25700 lr=6.000000e-04 loss=3.2090 ce=3.2090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.129891436197795e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034721377305686474, 'last_eos_top1': 0} +step=25750 lr=6.000000e-04 loss=2.5252 ce=2.5252 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2851878384244628e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037958091124892235, 'last_eos_top1': 0} +step=25800 lr=6.000000e-04 loss=2.7023 ce=2.7023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.803492654697038e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003416771534830332, 'last_eos_top1': 0} +step=25850 lr=6.000000e-04 loss=2.3172 ce=2.3172 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.288671283982694e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003685052040964365, 'last_eos_top1': 0} +step=25900 lr=6.000000e-04 loss=2.4755 ce=2.4755 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.585763963405043e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002250362653285265, 'last_eos_top1': 0} +step=25950 lr=6.000000e-04 loss=2.8682 ce=2.8682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8194663173053414e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024925037287175655, 'last_eos_top1': 0} +step=26000 lr=6.000000e-04 loss=2.5436 ce=2.5436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.166065238881856e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005371276754885912, 'last_eos_top1': 0} +step=26050 lr=6.000000e-04 loss=2.6651 ce=2.6651 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.769169102772139e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000840110587887466, 'last_eos_top1': 0} +step=26100 lr=6.000000e-04 loss=2.8757 ce=2.8757 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9216135115129873e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004109901492483914, 'last_eos_top1': 0} +step=26150 lr=6.000000e-04 loss=2.6536 ce=2.6536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9194558010203764e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022185146808624268, 'last_eos_top1': 0} +step=26200 lr=6.000000e-04 loss=2.7049 ce=2.7049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.063937997969333e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016693207435309887, 'last_eos_top1': 0} +step=26250 lr=6.000000e-04 loss=2.6023 ce=2.6023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2738626537611708e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007721815491095185, 'last_eos_top1': 0} +step=26300 lr=6.000000e-04 loss=3.2356 ce=3.2356 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0013085304526612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008987562032416463, 'last_eos_top1': 0} +step=26350 lr=6.000000e-04 loss=2.3575 ce=2.3575 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.355595825065393e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002148618455976248, 'last_eos_top1': 0} +step=26400 lr=6.000000e-04 loss=2.9439 ce=2.9439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5073741451487876e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007219764869660139, 'last_eos_top1': 0} +step=26450 lr=6.000000e-04 loss=2.5301 ce=2.5301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4422740605368745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002542549977079034, 'last_eos_top1': 0} +step=26500 lr=6.000000e-04 loss=2.8292 ce=2.8292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2225818611332215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012646829709410667, 'last_eos_top1': 0} +step=26550 lr=6.000000e-04 loss=2.2148 ce=2.2148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5173061910900287e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017940946854650974, 'last_eos_top1': 0} +step=26600 lr=6.000000e-04 loss=2.4072 ce=2.4072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4526472114084754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002195098903030157, 'last_eos_top1': 0} +step=26650 lr=6.000000e-04 loss=1.9943 ce=1.9943 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1773203368647955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015239715576171875, 'last_eos_top1': 0} +step=26700 lr=6.000000e-04 loss=2.6143 ce=2.6143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.859880441974383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004575912840664387, 'last_eos_top1': 0} +step=26750 lr=6.000000e-04 loss=2.3399 ce=2.3399 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.230345363495871e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004545957315713167, 'last_eos_top1': 0} +step=26800 lr=6.000000e-04 loss=2.8285 ce=2.8285 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.722034307429567e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008076871745288372, 'last_eos_top1': 0} +step=26850 lr=6.000000e-04 loss=2.3619 ce=2.3619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.260360270156525e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012375966180115938, 'last_eos_top1': 0} +step=26900 lr=6.000000e-04 loss=2.6174 ce=2.6174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2754862837027758e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001685870229266584, 'last_eos_top1': 0} +step=26950 lr=6.000000e-04 loss=2.1110 ce=2.1110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0895060768234544e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004305052105337381, 'last_eos_top1': 0} +step=27000 lr=6.000000e-04 loss=2.3814 ce=2.3814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5089030966919381e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031707854941487312, 'last_eos_top1': 0} +step=27050 lr=6.000000e-04 loss=2.5359 ce=2.5359 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5137078409898095e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002339364727959037, 'last_eos_top1': 0} +step=27100 lr=6.000000e-04 loss=2.4418 ce=2.4418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5262063243426383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001196403754875064, 'last_eos_top1': 0} +step=27150 lr=6.000000e-04 loss=3.4071 ce=3.4071 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6049343432532623e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019281465793028474, 'last_eos_top1': 0} +step=27200 lr=6.000000e-04 loss=2.1381 ce=2.1381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0612531696096994e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030709165148437023, 'last_eos_top1': 0} +step=27250 lr=6.000000e-04 loss=2.6766 ce=2.6766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6322297597071156e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024214317090809345, 'last_eos_top1': 0} +step=27300 lr=6.000000e-04 loss=2.8727 ce=2.8727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.30488788272487e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013864291831851006, 'last_eos_top1': 0} +step=27350 lr=6.000000e-04 loss=3.0092 ce=3.0092 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.108198016998358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001838029595091939, 'last_eos_top1': 0} +step=27400 lr=6.000000e-04 loss=2.6485 ce=2.6485 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.19260457722703e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003997692838311195, 'last_eos_top1': 0} +step=27450 lr=6.000000e-04 loss=2.5606 ce=2.5606 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.47314765653573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036047473549842834, 'last_eos_top1': 0} +step=27500 lr=6.000000e-04 loss=2.7229 ce=2.7229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.337594873504713e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007805095985531807, 'last_eos_top1': 0} +step=27550 lr=6.000000e-04 loss=2.1782 ce=2.1782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7558802685234696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014973541256040335, 'last_eos_top1': 0} +step=27600 lr=6.000000e-04 loss=2.3549 ce=2.3549 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4084122994972859e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001141594722867012, 'last_eos_top1': 0} +step=27650 lr=6.000000e-04 loss=2.8011 ce=2.8011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2837314898206387e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010508205741643906, 'last_eos_top1': 0} +step=27700 lr=6.000000e-04 loss=2.5098 ce=2.5098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0461916544009e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00404545571655035, 'last_eos_top1': 0} +step=27750 lr=6.000000e-04 loss=2.4777 ce=2.4777 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6126767150126398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002848324365913868, 'last_eos_top1': 0} +step=27800 lr=6.000000e-04 loss=2.6234 ce=2.6234 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6122714441735297e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005557074211537838, 'last_eos_top1': 0} +step=27850 lr=6.000000e-04 loss=3.0480 ce=3.0480 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.314243415137753e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002421901561319828, 'last_eos_top1': 0} +step=27900 lr=6.000000e-04 loss=2.3042 ce=2.3042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0870171283604577e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006032057572156191, 'last_eos_top1': 0} +step=27950 lr=6.000000e-04 loss=3.1893 ce=3.1893 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.094207204412669e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011529286857694387, 'last_eos_top1': 0} +step=28000 lr=6.000000e-04 loss=2.9402 ce=2.9402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.40969177664374e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011307460954412818, 'last_eos_top1': 0} +step=28050 lr=6.000000e-04 loss=2.4102 ce=2.4102 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.25072105624713e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00094818533398211, 'last_eos_top1': 0} +step=28100 lr=6.000000e-04 loss=2.9385 ce=2.9385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.125314444536343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020064935088157654, 'last_eos_top1': 0} +step=28150 lr=6.000000e-04 loss=2.7119 ce=2.7119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.467594433459453e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000891844101715833, 'last_eos_top1': 0} +step=28200 lr=6.000000e-04 loss=2.2375 ce=2.2375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6376230127643794e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011706643272191286, 'last_eos_top1': 0} +step=28250 lr=6.000000e-04 loss=2.2976 ce=2.2976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1704952309373766e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009636501199565828, 'last_eos_top1': 0} +step=28300 lr=6.000000e-04 loss=2.6024 ce=2.6024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3227526475675404e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007673032814636827, 'last_eos_top1': 0} +step=28350 lr=6.000000e-04 loss=2.7391 ce=2.7391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.840654633473605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001128688338212669, 'last_eos_top1': 0} +step=28400 lr=6.000000e-04 loss=2.5344 ce=2.5344 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6913881811196916e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015076971612870693, 'last_eos_top1': 0} +step=28450 lr=6.000000e-04 loss=2.8628 ce=2.8628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.285925751901232e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002847470575943589, 'last_eos_top1': 0} +step=28500 lr=6.000000e-04 loss=2.9515 ce=2.9515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5994092943146825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002558991312980652, 'last_eos_top1': 0} +step=28550 lr=6.000000e-04 loss=2.2433 ce=2.2433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.39753603131976e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012492910027503967, 'last_eos_top1': 0} +step=28600 lr=6.000000e-04 loss=2.7912 ce=2.7912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4411170670646243e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008207724895328283, 'last_eos_top1': 0} +step=28650 lr=6.000000e-04 loss=3.0647 ce=3.0647 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.35461149411276e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031317532993853092, 'last_eos_top1': 0} +step=28700 lr=6.000000e-04 loss=3.3671 ce=3.3671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2584172256756574e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009065067861229181, 'last_eos_top1': 0} +step=28750 lr=6.000000e-04 loss=2.0871 ce=2.0871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8733315048157237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010537388734519482, 'last_eos_top1': 0} +step=28800 lr=6.000000e-04 loss=2.7964 ce=2.7964 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.312093758722767e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013429650571197271, 'last_eos_top1': 0} +step=28850 lr=6.000000e-04 loss=2.4623 ce=2.4623 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.942175681004301e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0048768119886517525, 'last_eos_top1': 0} +step=28900 lr=6.000000e-04 loss=3.1374 ce=3.1374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.854677510797046e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004947120789438486, 'last_eos_top1': 0} +step=28950 lr=6.000000e-04 loss=2.4672 ce=2.4672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3730230421060696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035753706470131874, 'last_eos_top1': 0} +step=29000 lr=6.000000e-04 loss=2.4139 ce=2.4139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.615409463644028e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009142851922661066, 'last_eos_top1': 0} +step=29050 lr=6.000000e-04 loss=2.8520 ce=2.8520 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9272014216985554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012030155630782247, 'last_eos_top1': 0} +step=29100 lr=6.000000e-04 loss=2.4446 ce=2.4446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.915994966519065e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016291056526824832, 'last_eos_top1': 0} +step=29150 lr=6.000000e-04 loss=2.4598 ce=2.4598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.124467912130058e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018500983715057373, 'last_eos_top1': 0} +step=29200 lr=6.000000e-04 loss=2.4584 ce=2.4584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.194025627337396e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025901985354721546, 'last_eos_top1': 0} +step=29250 lr=6.000000e-04 loss=2.5658 ce=2.5658 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1809537429362535e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001342347008176148, 'last_eos_top1': 0} +step=29300 lr=6.000000e-04 loss=2.6375 ce=2.6375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.396526466938667e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023346291854977608, 'last_eos_top1': 0} +step=29350 lr=6.000000e-04 loss=2.6243 ce=2.6243 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.091260679066181e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003973749000579119, 'last_eos_top1': 0} +step=29400 lr=6.000000e-04 loss=2.3674 ce=2.3674 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5145753422984853e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008531114435754716, 'last_eos_top1': 0} +step=29450 lr=6.000000e-04 loss=2.3488 ce=2.3488 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.99420025839936e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001706180744804442, 'last_eos_top1': 0} +step=29500 lr=6.000000e-04 loss=2.3688 ce=2.3688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.143538924632594e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035548844607546926, 'last_eos_top1': 0} +step=29550 lr=6.000000e-04 loss=2.6444 ce=2.6444 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.301654942333698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027326550334692, 'last_eos_top1': 0} +step=29600 lr=6.000000e-04 loss=2.7911 ce=2.7911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.099808575119823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004569474142044783, 'last_eos_top1': 0} +step=29650 lr=6.000000e-04 loss=2.7584 ce=2.7584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1359897800721228e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00539788044989109, 'last_eos_top1': 0} +step=29700 lr=6.000000e-04 loss=2.6690 ce=2.6690 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5772365916054696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037653655745089054, 'last_eos_top1': 0} +step=29750 lr=6.000000e-04 loss=2.9409 ce=2.9409 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.840785939246416e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034493787679821253, 'last_eos_top1': 0} +step=29800 lr=6.000000e-04 loss=2.7622 ce=2.7622 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.858707663719542e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003762232605367899, 'last_eos_top1': 0} +step=29850 lr=6.000000e-04 loss=2.8666 ce=2.8666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.273124598810682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002904447726905346, 'last_eos_top1': 0} +step=29900 lr=6.000000e-04 loss=2.6913 ce=2.6913 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7463123842608184e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025820285081863403, 'last_eos_top1': 0} +step=29950 lr=6.000000e-04 loss=3.0341 ce=3.0341 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.67480502568651e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003001042176038027, 'last_eos_top1': 0} +step=30000 lr=6.000000e-04 loss=2.4958 ce=2.4958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4822969862725586e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013339563738554716, 'last_eos_top1': 0} +step=30050 lr=6.000000e-04 loss=2.0679 ce=2.0679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6846537619130686e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009779157117009163, 'last_eos_top1': 0} +step=30100 lr=6.000000e-04 loss=2.8084 ce=2.8084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.226478343596682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027765234699472785, 'last_eos_top1': 0} +step=30150 lr=6.000000e-04 loss=3.0575 ce=3.0575 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.307398426637519e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001216718228533864, 'last_eos_top1': 0} +step=30200 lr=6.000000e-04 loss=2.4311 ce=2.4311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6100264474516734e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002558867447078228, 'last_eos_top1': 0} +step=30250 lr=6.000000e-04 loss=2.3986 ce=2.3986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.457132566429209e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002755279652774334, 'last_eos_top1': 0} +step=30300 lr=6.000000e-04 loss=2.6966 ce=2.6966 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.626904905715492e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027812891639769077, 'last_eos_top1': 0} +step=30350 lr=6.000000e-04 loss=2.5084 ce=2.5084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.102026337524876e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012071493547409773, 'last_eos_top1': 0} +step=30400 lr=6.000000e-04 loss=2.6341 ce=2.6341 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.776063178724144e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021466391626745462, 'last_eos_top1': 0} +step=30450 lr=6.000000e-04 loss=2.5938 ce=2.5938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1899814126081765e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000950571324210614, 'last_eos_top1': 0} +step=30500 lr=6.000000e-04 loss=2.4450 ce=2.4450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.949175025918521e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002047289162874222, 'last_eos_top1': 0} +step=30550 lr=6.000000e-04 loss=2.2430 ce=2.2430 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0624482203857042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004594244994223118, 'last_eos_top1': 0} +step=30600 lr=6.000000e-04 loss=2.6826 ce=2.6826 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.951217564055696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006239769048988819, 'last_eos_top1': 0} +step=30650 lr=6.000000e-04 loss=3.0798 ce=3.0798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6902659707702696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00996433012187481, 'last_eos_top1': 0} +step=30700 lr=6.000000e-04 loss=2.8510 ce=2.8510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.10969480779022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0116948913782835, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=500000 rows=491809 buffered=8191 part=openwebtext-t5 +step=30750 lr=6.000000e-04 loss=1.9626 ce=1.9626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.292611861776095e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005553939379751682, 'last_eos_top1': 0} +step=30800 lr=6.000000e-04 loss=3.0394 ce=3.0394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2651431461563334e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006542573682963848, 'last_eos_top1': 0} +step=30850 lr=6.000000e-04 loss=2.5433 ce=2.5433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.400154673727229e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007835425436496735, 'last_eos_top1': 0} +step=30900 lr=6.000000e-04 loss=3.5822 ce=3.5822 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9835959139745682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015085028717294335, 'last_eos_top1': 0} +step=30950 lr=6.000000e-04 loss=2.7775 ce=2.7775 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8005634046858177e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018932389793917537, 'last_eos_top1': 0} +step=31000 lr=6.000000e-04 loss=2.2045 ce=2.2045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.331373803433962e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005198941566050053, 'last_eos_top1': 0} +step=31050 lr=6.000000e-04 loss=2.7628 ce=2.7628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.637121386011131e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008210432715713978, 'last_eos_top1': 0} +step=31100 lr=6.000000e-04 loss=2.2985 ce=2.2985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2605711162905209e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028950502164661884, 'last_eos_top1': 0} +step=31150 lr=6.000000e-04 loss=2.9230 ce=2.9230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0186895451624878e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009380761533975601, 'last_eos_top1': 0} +step=31200 lr=6.000000e-04 loss=3.1324 ce=3.1324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7697591829346493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007211153861135244, 'last_eos_top1': 0} +step=31250 lr=6.000000e-04 loss=1.9759 ce=1.9759 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.009776289924048e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0116449985653162, 'last_eos_top1': 0} +step=31300 lr=6.000000e-04 loss=2.3447 ce=2.3447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9555667424574494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007421958260238171, 'last_eos_top1': 0} +step=31350 lr=6.000000e-04 loss=2.5231 ce=2.5231 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3868022253736854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008204606361687183, 'last_eos_top1': 0} +step=31400 lr=6.000000e-04 loss=2.9032 ce=2.9032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9545710529200733e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005528750829398632, 'last_eos_top1': 0} +step=31450 lr=6.000000e-04 loss=2.5733 ce=2.5733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.133153637056239e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003252234775573015, 'last_eos_top1': 0} +step=31500 lr=6.000000e-04 loss=2.2915 ce=2.2915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.683166925038677e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011900055687874556, 'last_eos_top1': 0} +step=31550 lr=6.000000e-04 loss=2.5848 ce=2.5848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.132149191107601e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018343235133215785, 'last_eos_top1': 0} +step=31600 lr=6.000000e-04 loss=2.3574 ce=2.3574 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.62817661173176e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0066267093643546104, 'last_eos_top1': 0} +step=31650 lr=6.000000e-04 loss=2.6485 ce=2.6485 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5242034351103939e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005190770607441664, 'last_eos_top1': 0} +step=31700 lr=6.000000e-04 loss=2.5080 ce=2.5080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.785874701454304e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00292058102786541, 'last_eos_top1': 0} +step=31750 lr=6.000000e-04 loss=2.6879 ce=2.6879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.686225682031363e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001422244356945157, 'last_eos_top1': 0} +step=31800 lr=6.000000e-04 loss=2.9498 ce=2.9498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1941207958443556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00025166990235447884, 'last_eos_top1': 0} +step=31850 lr=6.000000e-04 loss=1.9827 ce=1.9827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3404172023001593e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019904735381715, 'last_eos_top1': 0} +step=31900 lr=6.000000e-04 loss=1.7593 ce=1.7593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5728048058226705e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000651203328743577, 'last_eos_top1': 0} +step=31950 lr=6.000000e-04 loss=2.9474 ce=2.9474 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0985378796467558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013727662153542042, 'last_eos_top1': 0} +step=32000 lr=6.000000e-04 loss=2.4511 ce=2.4511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.18284242489608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006245485856197774, 'last_eos_top1': 0} +step=32050 lr=6.000000e-04 loss=2.3075 ce=2.3075 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.121163172181696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007334968540817499, 'last_eos_top1': 0} +step=32100 lr=6.000000e-04 loss=2.3110 ce=2.3110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.637061258312315e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012112765107303858, 'last_eos_top1': 0} +step=32150 lr=6.000000e-04 loss=2.4860 ce=2.4860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.929688369273208e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004351342795416713, 'last_eos_top1': 0} +step=32200 lr=6.000000e-04 loss=2.9180 ce=2.9180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.94236017705407e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030523232999257743, 'last_eos_top1': 0} +step=32250 lr=6.000000e-04 loss=2.5362 ce=2.5362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.236528184264898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008822265663184226, 'last_eos_top1': 0} +step=32300 lr=6.000000e-04 loss=3.0123 ce=3.0123 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1633833966916427e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006259407382458448, 'last_eos_top1': 0} +step=32350 lr=6.000000e-04 loss=3.0186 ce=3.0186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.986619463423267e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033722532680258155, 'last_eos_top1': 0} +step=32400 lr=6.000000e-04 loss=2.5742 ce=2.5742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7403152014594525e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003851515066344291, 'last_eos_top1': 0} +step=32450 lr=6.000000e-04 loss=2.4777 ce=2.4777 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.87670266313944e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000449139071861282, 'last_eos_top1': 0} +step=32500 lr=6.000000e-04 loss=3.1806 ce=3.1806 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6970843716990203e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039932652725838125, 'last_eos_top1': 0} +step=32550 lr=6.000000e-04 loss=2.1419 ce=2.1419 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.858853647718206e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005377658177167177, 'last_eos_top1': 0} +step=32600 lr=6.000000e-04 loss=2.6207 ce=2.6207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1298069694021251e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018379564862698317, 'last_eos_top1': 0} +step=32650 lr=6.000000e-04 loss=2.8151 ce=2.8151 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5290654118871316e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011172685772180557, 'last_eos_top1': 0} +step=32700 lr=6.000000e-04 loss=2.6193 ce=2.6193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6938161934376694e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005342225776985288, 'last_eos_top1': 0} +step=32750 lr=6.000000e-04 loss=2.5732 ce=2.5732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8135651771444827e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003607456455938518, 'last_eos_top1': 0} +step=32800 lr=6.000000e-04 loss=2.4073 ce=2.4073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9221264665247872e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028731959173455834, 'last_eos_top1': 0} +step=32850 lr=6.000000e-04 loss=2.7668 ce=2.7668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3800579836242832e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003190545248799026, 'last_eos_top1': 0} +step=32900 lr=6.000000e-04 loss=2.6451 ce=2.6451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8873408407671377e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005055592628195882, 'last_eos_top1': 0} +step=32950 lr=6.000000e-04 loss=2.4012 ce=2.4012 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0757859601872042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001405392773449421, 'last_eos_top1': 0} +step=33000 lr=6.000000e-04 loss=2.7921 ce=2.7921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.32180120999692e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000653325580060482, 'last_eos_top1': 0} +step=33050 lr=6.000000e-04 loss=2.9923 ce=2.9923 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2027058952953666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004906575195491314, 'last_eos_top1': 0} +step=33100 lr=6.000000e-04 loss=2.5585 ce=2.5585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4232561524258927e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004219766706228256, 'last_eos_top1': 0} +step=33150 lr=6.000000e-04 loss=2.2849 ce=2.2849 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2867094483226538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004166822414845228, 'last_eos_top1': 0} +step=33200 lr=6.000000e-04 loss=2.7743 ce=2.7743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0278313311573584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003202090971171856, 'last_eos_top1': 0} +step=33250 lr=6.000000e-04 loss=2.6642 ce=2.6642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4917028238414787e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002833245787769556, 'last_eos_top1': 0} +step=33300 lr=6.000000e-04 loss=2.2635 ce=2.2635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3892742319731042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002064388943836093, 'last_eos_top1': 0} +step=33350 lr=6.000000e-04 loss=2.3386 ce=2.3386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.562176475999877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003890711348503828, 'last_eos_top1': 0} +step=33400 lr=6.000000e-04 loss=2.2594 ce=2.2594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.260196748073213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001253552851267159, 'last_eos_top1': 0} +step=33450 lr=6.000000e-04 loss=2.1573 ce=2.1573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0673275028239004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004897666512988508, 'last_eos_top1': 0} +step=33500 lr=6.000000e-04 loss=2.2773 ce=2.2773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0433615898364224e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004628580994904041, 'last_eos_top1': 0} +step=33550 lr=6.000000e-04 loss=2.8064 ce=2.8064 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.248688178951852e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016598443035036325, 'last_eos_top1': 0} +step=33600 lr=6.000000e-04 loss=2.7510 ce=2.7510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.837255695136264e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004263291135430336, 'last_eos_top1': 0} +step=33650 lr=6.000000e-04 loss=2.9364 ce=2.9364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.864666647743434e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005018551833927631, 'last_eos_top1': 0} +step=33700 lr=6.000000e-04 loss=2.3443 ce=2.3443 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0835752366110682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002960829995572567, 'last_eos_top1': 0} +step=33750 lr=6.000000e-04 loss=2.4170 ce=2.4170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7416579036507756e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005418038344942033, 'last_eos_top1': 0} +step=33800 lr=6.000000e-04 loss=2.5024 ce=2.5024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7450452762423083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002035992918536067, 'last_eos_top1': 0} +step=33850 lr=6.000000e-04 loss=2.5275 ce=2.5275 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7599799068411812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005468269810080528, 'last_eos_top1': 0} +step=33900 lr=6.000000e-04 loss=2.7616 ce=2.7616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6810248527908698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001076048007234931, 'last_eos_top1': 0} +step=33950 lr=6.000000e-04 loss=2.3761 ce=2.3761 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4654975529992953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021498496644198895, 'last_eos_top1': 0} +step=34000 lr=6.000000e-04 loss=2.0057 ce=2.0057 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4293272872455418e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001328582176938653, 'last_eos_top1': 0} +step=34050 lr=6.000000e-04 loss=2.4996 ce=2.4996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3210823453846388e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005189744406379759, 'last_eos_top1': 0} +step=34100 lr=6.000000e-04 loss=2.8441 ce=2.8441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.794146197033115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002603194734547287, 'last_eos_top1': 0} +step=34150 lr=6.000000e-04 loss=2.5102 ce=2.5102 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1175206964253448e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028328015469014645, 'last_eos_top1': 0} +step=34200 lr=6.000000e-04 loss=2.9329 ce=2.9329 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3033801224082708e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004138246877118945, 'last_eos_top1': 0} +step=34250 lr=6.000000e-04 loss=2.4713 ce=2.4713 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7527510105283e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.414515519281849e-05, 'last_eos_top1': 0} +step=34300 lr=6.000000e-04 loss=2.9226 ce=2.9226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3457439965568483e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029245048062875867, 'last_eos_top1': 0} +step=34350 lr=6.000000e-04 loss=2.6065 ce=2.6065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.240506571775768e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032736771390773356, 'last_eos_top1': 0} +step=34400 lr=6.000000e-04 loss=2.7176 ce=2.7176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.992395694600418e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030962532036937773, 'last_eos_top1': 0} +step=34450 lr=6.000000e-04 loss=2.4151 ce=2.4151 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.500422669982072e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015026838518679142, 'last_eos_top1': 0} +step=34500 lr=6.000000e-04 loss=2.7624 ce=2.7624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.114384738495573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001279604621231556, 'last_eos_top1': 0} +step=34550 lr=6.000000e-04 loss=2.4965 ce=2.4965 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2175840311101638e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012617885367944837, 'last_eos_top1': 0} +step=34600 lr=6.000000e-04 loss=2.3539 ce=2.3539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.663932824973017e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005899358657188714, 'last_eos_top1': 0} +step=34650 lr=6.000000e-04 loss=2.5281 ce=2.5281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.218432539142668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036773516330868006, 'last_eos_top1': 0} +step=34700 lr=6.000000e-04 loss=2.9355 ce=2.9355 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.156455255113542e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001777212310116738, 'last_eos_top1': 0} +step=34750 lr=6.000000e-04 loss=2.5615 ce=2.5615 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9626569812535308e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009966053767129779, 'last_eos_top1': 0} +step=34800 lr=6.000000e-04 loss=2.7960 ce=2.7960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6564947145525366e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003506283974274993, 'last_eos_top1': 0} +step=34850 lr=6.000000e-04 loss=2.0979 ce=2.0979 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7200665752170607e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008930120384320617, 'last_eos_top1': 0} +step=34900 lr=6.000000e-04 loss=2.6848 ce=2.6848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.052084012888372e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001318307826295495, 'last_eos_top1': 0} +step=34950 lr=6.000000e-04 loss=2.5508 ce=2.5508 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.659079296397977e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021217753237579018, 'last_eos_top1': 0} +step=35000 lr=6.000000e-04 loss=2.1173 ce=2.1173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3205974937591236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005387855344451964, 'last_eos_top1': 0} +step=35050 lr=6.000000e-04 loss=2.6192 ce=2.6192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.057533129118383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005291915498673916, 'last_eos_top1': 0} +step=35100 lr=6.000000e-04 loss=2.7869 ce=2.7869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8793813069351017e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005622395314276218, 'last_eos_top1': 0} +step=35150 lr=6.000000e-04 loss=3.0811 ce=3.0811 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2879961761645973e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018570057582110167, 'last_eos_top1': 0} +step=35200 lr=6.000000e-04 loss=2.7517 ce=2.7517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5711769922054373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006703869439661503, 'last_eos_top1': 0} +step=35250 lr=6.000000e-04 loss=2.1679 ce=2.1679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3699096345808357e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005585352191701531, 'last_eos_top1': 0} +step=35300 lr=6.000000e-04 loss=2.9737 ce=2.9737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.090580528602004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004478638991713524, 'last_eos_top1': 0} +step=35350 lr=6.000000e-04 loss=2.5691 ce=2.5691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4638909306086134e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00025902202469296753, 'last_eos_top1': 0} +step=35400 lr=6.000000e-04 loss=2.6650 ce=2.6650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0086921975016594e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002854868653230369, 'last_eos_top1': 0} +step=35450 lr=6.000000e-04 loss=2.7054 ce=2.7054 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.518201265251264e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007446779636666179, 'last_eos_top1': 0} +step=35500 lr=6.000000e-04 loss=2.4476 ce=2.4476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0791960448841564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020087769371457398, 'last_eos_top1': 0} +step=35550 lr=6.000000e-04 loss=2.2128 ce=2.2128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2622074791579507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001954547129571438, 'last_eos_top1': 0} +step=35600 lr=6.000000e-04 loss=2.1763 ce=2.1763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6892386560793966e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001769036753103137, 'last_eos_top1': 0} +step=35650 lr=6.000000e-04 loss=2.3743 ce=2.3743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4377546651521698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016695449594408274, 'last_eos_top1': 0} +step=35700 lr=6.000000e-04 loss=2.7095 ce=2.7095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.185834793839604e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002562887966632843, 'last_eos_top1': 0} +step=35750 lr=6.000000e-04 loss=2.6693 ce=2.6693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.834296199376695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021930501679889858, 'last_eos_top1': 0} +step=35800 lr=6.000000e-04 loss=2.8334 ce=2.8334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2564012649818324e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015342514961957932, 'last_eos_top1': 0} +step=35850 lr=6.000000e-04 loss=2.6650 ce=2.6650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6239580367691815e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003966851159930229, 'last_eos_top1': 0} +step=35900 lr=6.000000e-04 loss=2.8271 ce=2.8271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2195566998561844e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004785036144312471, 'last_eos_top1': 0} +step=35950 lr=6.000000e-04 loss=2.3967 ce=2.3967 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3849557944922708e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035852326545864344, 'last_eos_top1': 0} +step=36000 lr=6.000000e-04 loss=2.3864 ce=2.3864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.398453595262254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003779046004638076, 'last_eos_top1': 0} +step=36050 lr=6.000000e-04 loss=2.4143 ce=2.4143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9037302990909666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004507254809141159, 'last_eos_top1': 0} +step=36100 lr=6.000000e-04 loss=2.4418 ce=2.4418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.384697477798909e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019243574934080243, 'last_eos_top1': 0} +step=36150 lr=6.000000e-04 loss=2.4179 ce=2.4179 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1714251741068438e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027198082534596324, 'last_eos_top1': 0} +step=36200 lr=6.000000e-04 loss=2.7606 ce=2.7606 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.83775090490235e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033929134951904416, 'last_eos_top1': 0} +step=36250 lr=6.000000e-04 loss=2.0314 ce=2.0314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.457651463482762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001376032450934872, 'last_eos_top1': 0} +step=36300 lr=6.000000e-04 loss=2.8117 ce=2.8117 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.506742571829818e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013522325025405735, 'last_eos_top1': 0} +step=36350 lr=6.000000e-04 loss=2.7618 ce=2.7618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.318254342360888e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006460865843109787, 'last_eos_top1': 0} +step=36400 lr=6.000000e-04 loss=2.3275 ce=2.3275 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4179281606629957e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032328180968761444, 'last_eos_top1': 0} +step=36450 lr=6.000000e-04 loss=3.0112 ce=3.0112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3231814591563307e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002706219907850027, 'last_eos_top1': 0} +step=36500 lr=6.000000e-04 loss=2.1041 ce=2.1041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.04998793581035e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003663539420813322, 'last_eos_top1': 0} +step=36550 lr=6.000000e-04 loss=2.9086 ce=2.9086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.34944111778168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008199589792639017, 'last_eos_top1': 0} +step=36600 lr=6.000000e-04 loss=2.5938 ce=2.5938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.338153717573732e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015950704691931605, 'last_eos_top1': 0} +step=36650 lr=6.000000e-04 loss=2.2819 ce=2.2819 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7469467972405255e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020221639424562454, 'last_eos_top1': 0} +step=36700 lr=6.000000e-04 loss=3.0456 ce=3.0456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9521019061794505e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001832747831940651, 'last_eos_top1': 0} +step=36750 lr=6.000000e-04 loss=2.9119 ce=2.9119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.50246848736424e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032225465402007103, 'last_eos_top1': 0} +step=36800 lr=6.000000e-04 loss=2.4712 ce=2.4712 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1969342924421653e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038522519171237946, 'last_eos_top1': 0} +step=36850 lr=6.000000e-04 loss=2.5872 ce=2.5872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4620845831814222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029841664945706725, 'last_eos_top1': 0} +step=36900 lr=6.000000e-04 loss=2.6594 ce=2.6594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5448284658486955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.619161962997168e-05, 'last_eos_top1': 0} +step=36950 lr=6.000000e-04 loss=2.9060 ce=2.9060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.27385201267316e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002146981714759022, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=3] docs=600000 rows=591809 buffered=8191 part=openwebtext-t5 +step=37000 lr=6.000000e-04 loss=2.0683 ce=2.0683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.290376172226388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002747515682131052, 'last_eos_top1': 0} +step=37050 lr=6.000000e-04 loss=2.3139 ce=2.3139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8017917682300322e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006215405883267522, 'last_eos_top1': 0} +step=37100 lr=6.000000e-04 loss=2.7525 ce=2.7525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3363953561056405e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00041569466702640057, 'last_eos_top1': 0} +step=37150 lr=6.000000e-04 loss=2.9568 ce=2.9568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7404465324943885e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007273862138390541, 'last_eos_top1': 0} +step=37200 lr=6.000000e-04 loss=3.2441 ce=3.2441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7781008611782454e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012881718575954437, 'last_eos_top1': 0} +step=37250 lr=6.000000e-04 loss=2.6834 ce=2.6834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4303793250292074e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007904077647253871, 'last_eos_top1': 0} +step=37300 lr=6.000000e-04 loss=2.5387 ce=2.5387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.65280798682943e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014949124306440353, 'last_eos_top1': 0} +step=37350 lr=6.000000e-04 loss=1.9695 ce=1.9695 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1168455785082188e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003169241826981306, 'last_eos_top1': 0} +step=37400 lr=6.000000e-04 loss=2.4321 ce=2.4321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3852244592271745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007461474742740393, 'last_eos_top1': 0} +step=37450 lr=6.000000e-04 loss=2.5569 ce=2.5569 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.036288813338615e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000604486558586359, 'last_eos_top1': 0} +step=37500 lr=6.000000e-04 loss=2.1914 ce=2.1914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.781642802408896e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016787939239293337, 'last_eos_top1': 0} +step=37550 lr=6.000000e-04 loss=2.7091 ce=2.7091 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0249967090203427e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017150866333395243, 'last_eos_top1': 0} +step=37600 lr=6.000000e-04 loss=2.6508 ce=2.6508 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1194158762227744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009568817913532257, 'last_eos_top1': 0} +step=37650 lr=6.000000e-04 loss=2.1156 ce=2.1156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.187009108136408e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037567061372101307, 'last_eos_top1': 0} +step=37700 lr=6.000000e-04 loss=3.1577 ce=3.1577 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3161527022020891e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002093713032081723, 'last_eos_top1': 0} +step=37750 lr=6.000000e-04 loss=2.5597 ce=2.5597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8649836420081556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042505940655246377, 'last_eos_top1': 0} +step=37800 lr=6.000000e-04 loss=2.5840 ce=2.5840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.226893775514327e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003625987097620964, 'last_eos_top1': 0} +step=37850 lr=6.000000e-04 loss=2.5461 ce=2.5461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.886141924653202e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032848368864506483, 'last_eos_top1': 0} +step=37900 lr=6.000000e-04 loss=2.7676 ce=2.7676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9853152480209246e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00280018150806427, 'last_eos_top1': 0} +step=37950 lr=6.000000e-04 loss=2.2323 ce=2.2323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.020005796221085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004774635657668114, 'last_eos_top1': 0} +step=38000 lr=6.000000e-04 loss=3.0499 ce=3.0499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.867751951882383e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003587225452065468, 'last_eos_top1': 0} +step=38050 lr=6.000000e-04 loss=2.4027 ce=2.4027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.025485501799267e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002827730495482683, 'last_eos_top1': 0} +step=38100 lr=6.000000e-04 loss=2.8694 ce=2.8694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.226113767595962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0050632343627512455, 'last_eos_top1': 0} +step=38150 lr=6.000000e-04 loss=2.4362 ce=2.4362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.800780592020601e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018177026649937034, 'last_eos_top1': 0} +step=38200 lr=6.000000e-04 loss=2.4587 ce=2.4587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6885078113991767e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030201231129467487, 'last_eos_top1': 0} +step=38250 lr=6.000000e-04 loss=2.5786 ce=2.5786 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.045848304987885e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004966901149600744, 'last_eos_top1': 0} +step=38300 lr=6.000000e-04 loss=2.7123 ce=2.7123 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.506424248684198e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006915146950632334, 'last_eos_top1': 0} +step=38350 lr=6.000000e-04 loss=2.3272 ce=2.3272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6246136258123443e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017688095103949308, 'last_eos_top1': 0} +step=38400 lr=6.000000e-04 loss=2.6294 ce=2.6294 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2674787032883614e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039161875611171126, 'last_eos_top1': 0} +step=38450 lr=6.000000e-04 loss=2.2494 ce=2.2494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3669397958437912e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004776868736371398, 'last_eos_top1': 0} +step=38500 lr=6.000000e-04 loss=2.2339 ce=2.2339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6895148291951045e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007557852659374475, 'last_eos_top1': 0} +step=38550 lr=6.000000e-04 loss=2.7917 ce=2.7917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1451319955522195e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008591357618570328, 'last_eos_top1': 0} +step=38600 lr=6.000000e-04 loss=2.5898 ce=2.5898 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2290430933935568e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011284598149359226, 'last_eos_top1': 0} +step=38650 lr=6.000000e-04 loss=2.7889 ce=2.7889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8892271327786148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016750013455748558, 'last_eos_top1': 0} +step=38700 lr=6.000000e-04 loss=3.0312 ce=3.0312 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6908812540350482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031131571158766747, 'last_eos_top1': 0} +step=38750 lr=6.000000e-04 loss=2.3056 ce=2.3056 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0247947506722994e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023109130561351776, 'last_eos_top1': 0} +step=38800 lr=6.000000e-04 loss=2.4785 ce=2.4785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.083500632783398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010171375470235944, 'last_eos_top1': 0} +step=38850 lr=6.000000e-04 loss=2.3097 ce=2.3097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.549214150349144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006062077009119093, 'last_eos_top1': 0} +step=38900 lr=6.000000e-04 loss=2.3487 ce=2.3487 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.860845809977036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029425189131870866, 'last_eos_top1': 0} +step=38950 lr=6.000000e-04 loss=2.1172 ce=2.1172 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2282107238424942e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010701421415433288, 'last_eos_top1': 0} +step=39000 lr=6.000000e-04 loss=2.5829 ce=2.5829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1694775568903424e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005906621227040887, 'last_eos_top1': 0} +step=39050 lr=6.000000e-04 loss=2.4851 ce=2.4851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7533806385472417e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008688722737133503, 'last_eos_top1': 0} +step=39100 lr=6.000000e-04 loss=2.8337 ce=2.8337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6036397937568836e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017694359412416816, 'last_eos_top1': 0} +step=39150 lr=6.000000e-04 loss=2.7774 ce=2.7774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.011737524298951e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015544724883511662, 'last_eos_top1': 0} +step=39200 lr=6.000000e-04 loss=2.8871 ce=2.8871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.803381696343422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006118151359260082, 'last_eos_top1': 0} +step=39250 lr=6.000000e-04 loss=2.5036 ce=2.5036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6539608370512724e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006251696613617241, 'last_eos_top1': 0} +step=39300 lr=6.000000e-04 loss=2.6477 ce=2.6477 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.069947681273334e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009277501376345754, 'last_eos_top1': 0} +step=39350 lr=6.000000e-04 loss=3.1509 ce=3.1509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.525254629086703e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039450329495593905, 'last_eos_top1': 0} +step=39400 lr=6.000000e-04 loss=2.6230 ce=2.6230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.13206151581835e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003468449576757848, 'last_eos_top1': 0} +step=39450 lr=6.000000e-04 loss=3.1725 ce=3.1725 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9125314793200232e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005046594887971878, 'last_eos_top1': 0} +step=39500 lr=6.000000e-04 loss=2.6871 ce=2.6871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.810534558899235e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005690834950655699, 'last_eos_top1': 0} +step=39550 lr=6.000000e-04 loss=2.9052 ce=2.9052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.078835319087375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003684868570417166, 'last_eos_top1': 0} +step=39600 lr=6.000000e-04 loss=2.2822 ce=2.2822 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9888095266651362e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006892859237268567, 'last_eos_top1': 0} +step=39650 lr=6.000000e-04 loss=2.7290 ce=2.7290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.166371268685907e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006477520801126957, 'last_eos_top1': 0} +step=39700 lr=6.000000e-04 loss=2.0403 ce=2.0403 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5079501281434204e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005265206564217806, 'last_eos_top1': 0} +step=39750 lr=6.000000e-04 loss=2.5162 ce=2.5162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.054587664839346e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001238950644619763, 'last_eos_top1': 0} +step=39800 lr=6.000000e-04 loss=2.1036 ce=2.1036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.621550472918898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000845115864649415, 'last_eos_top1': 0} +step=39850 lr=6.000000e-04 loss=2.0626 ce=2.0626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0322542695794255e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017279388848692179, 'last_eos_top1': 0} +step=39900 lr=6.000000e-04 loss=2.5122 ce=2.5122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4603909221477807e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017095349030569196, 'last_eos_top1': 0} +step=39950 lr=6.000000e-04 loss=2.4592 ce=2.4592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.818078979151323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005883989506401122, 'last_eos_top1': 0} +step=40000 lr=6.000000e-04 loss=2.7124 ce=2.7124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7825033359695226e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013398622395470738, 'last_eos_top1': 0} +step=40050 lr=6.000000e-04 loss=2.4237 ce=2.4237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.231588561902754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001413355115801096, 'last_eos_top1': 0} +step=40100 lr=6.000000e-04 loss=2.4011 ce=2.4011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0746472475584596e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016056518070399761, 'last_eos_top1': 0} +step=40150 lr=6.000000e-04 loss=2.9618 ce=2.9618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3927147847425658e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002126433653756976, 'last_eos_top1': 0} +step=40200 lr=6.000000e-04 loss=2.6471 ce=2.6471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4597746232757345e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027793957851827145, 'last_eos_top1': 0} +step=40250 lr=6.000000e-04 loss=2.2428 ce=2.2428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0436909835552797e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024334643967449665, 'last_eos_top1': 0} +step=40300 lr=6.000000e-04 loss=2.6568 ce=2.6568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3991910236654803e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011156961554661393, 'last_eos_top1': 0} +step=40350 lr=6.000000e-04 loss=2.4140 ce=2.4140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2950818674871698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006969514070078731, 'last_eos_top1': 0} +step=40400 lr=6.000000e-04 loss=2.1386 ce=2.1386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0779501432552934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002023105975240469, 'last_eos_top1': 0} +step=40450 lr=6.000000e-04 loss=1.8269 ce=1.8269 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.526974665466696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006140035111457109, 'last_eos_top1': 0} +step=40500 lr=6.000000e-04 loss=3.0108 ce=3.0108 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.586982216394972e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007753531681373715, 'last_eos_top1': 0} +step=40550 lr=6.000000e-04 loss=2.2039 ce=2.2039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.245459498022683e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010626590810716152, 'last_eos_top1': 0} +step=40600 lr=6.000000e-04 loss=2.3322 ce=2.3322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1211179046076722e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018420638516545296, 'last_eos_top1': 0} +step=40650 lr=6.000000e-04 loss=2.6494 ce=2.6494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2869671991211362e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018093249527737498, 'last_eos_top1': 0} +step=40700 lr=6.000000e-04 loss=2.4036 ce=2.4036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.022194717545062e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006873573292978108, 'last_eos_top1': 0} +step=40750 lr=6.000000e-04 loss=2.9358 ce=2.9358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.338119909632951e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000427233666414395, 'last_eos_top1': 0} +step=40800 lr=6.000000e-04 loss=2.7538 ce=2.7538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.676828656462021e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031116436002776027, 'last_eos_top1': 0} +step=40850 lr=6.000000e-04 loss=2.7228 ce=2.7228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.735181497992016e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007131886668503284, 'last_eos_top1': 0} +step=40900 lr=6.000000e-04 loss=2.2717 ce=2.2717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0533735803619493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003097417065873742, 'last_eos_top1': 0} +step=40950 lr=6.000000e-04 loss=2.3864 ce=2.3864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.818483365757857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006075972924008965, 'last_eos_top1': 0} +step=41000 lr=6.000000e-04 loss=2.7769 ce=2.7769 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.914898222021293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004994641058146954, 'last_eos_top1': 0} +step=41050 lr=6.000000e-04 loss=2.5762 ce=2.5762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5686855476815253e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012491834349930286, 'last_eos_top1': 0} +step=41100 lr=6.000000e-04 loss=2.8673 ce=2.8673 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.560314538248349e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024007301544770598, 'last_eos_top1': 0} +step=41150 lr=6.000000e-04 loss=2.3258 ce=2.3258 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.74145677901106e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014904301497153938, 'last_eos_top1': 0} +step=41200 lr=6.000000e-04 loss=2.7731 ce=2.7731 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.372140806983225e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000381452904548496, 'last_eos_top1': 0} +step=41250 lr=6.000000e-04 loss=2.6173 ce=2.6173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0108163628028706e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003919232403859496, 'last_eos_top1': 0} +step=41300 lr=6.000000e-04 loss=2.3670 ce=2.3670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.112948252237402e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004129334120079875, 'last_eos_top1': 0} +step=41350 lr=6.000000e-04 loss=2.8308 ce=2.8308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1939207979594357e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011390974977985024, 'last_eos_top1': 0} +step=41400 lr=6.000000e-04 loss=2.4192 ce=2.4192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.621275623620022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015510747907683253, 'last_eos_top1': 0} +step=41450 lr=6.000000e-04 loss=2.9061 ce=2.9061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.82114306173753e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024447417818009853, 'last_eos_top1': 0} +step=41500 lr=6.000000e-04 loss=2.1839 ce=2.1839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9124719983665273e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019724685698747635, 'last_eos_top1': 0} +step=41550 lr=6.000000e-04 loss=2.8349 ce=2.8349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1544854462263174e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010693180374801159, 'last_eos_top1': 0} +step=41600 lr=6.000000e-04 loss=1.9122 ce=1.9122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6083938791998662e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006474233814515173, 'last_eos_top1': 0} +step=41650 lr=6.000000e-04 loss=2.9982 ce=2.9982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4534914953401312e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003336862428113818, 'last_eos_top1': 0} +step=41700 lr=6.000000e-04 loss=2.8748 ce=2.8748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.009188266238198e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020977057283744216, 'last_eos_top1': 0} +step=41750 lr=6.000000e-04 loss=2.4544 ce=2.4544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.739680091210175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040514685679227114, 'last_eos_top1': 0} +step=41800 lr=6.000000e-04 loss=2.3062 ce=2.3062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.351280116068665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005175097612664104, 'last_eos_top1': 0} +step=41850 lr=6.000000e-04 loss=2.7989 ce=2.7989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.330374395067338e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003770890471059829, 'last_eos_top1': 0} +step=41900 lr=6.000000e-04 loss=2.8575 ce=2.8575 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.534885869768914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000627861765678972, 'last_eos_top1': 0} +step=41950 lr=6.000000e-04 loss=2.3730 ce=2.3730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.830684777814895e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008201635209843516, 'last_eos_top1': 0} +step=42000 lr=6.000000e-04 loss=2.8652 ce=2.8652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3847782611264847e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015240460634231567, 'last_eos_top1': 0} +step=42050 lr=6.000000e-04 loss=2.8856 ce=2.8856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2346341463853605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007377985748462379, 'last_eos_top1': 0} +step=42100 lr=6.000000e-04 loss=3.1400 ce=3.1400 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1177476810407825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008640222367830575, 'last_eos_top1': 0} +step=42150 lr=6.000000e-04 loss=2.7761 ce=2.7761 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.006572660524398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005404999246820807, 'last_eos_top1': 0} +step=42200 lr=6.000000e-04 loss=2.5189 ce=2.5189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1132864528917708e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006121157202869654, 'last_eos_top1': 0} +step=42250 lr=6.000000e-04 loss=2.5369 ce=2.5369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9555467588361353e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006450818618759513, 'last_eos_top1': 0} +step=42300 lr=6.000000e-04 loss=2.3368 ce=2.3368 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.752352131414227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010063308291137218, 'last_eos_top1': 0} +step=42350 lr=6.000000e-04 loss=2.4542 ce=2.4542 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.341210559592582e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005621608579531312, 'last_eos_top1': 0} +step=42400 lr=6.000000e-04 loss=3.3359 ce=3.3359 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9804818293778226e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010090485448017716, 'last_eos_top1': 0} +step=42450 lr=6.000000e-04 loss=2.7715 ce=2.7715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0126613233296666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010012747952714562, 'last_eos_top1': 0} +step=42500 lr=6.000000e-04 loss=2.9740 ce=2.9740 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.190685336245224e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009693456813693047, 'last_eos_top1': 0} +step=42550 lr=6.000000e-04 loss=1.9788 ce=1.9788 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6331003280356526e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006300105014815927, 'last_eos_top1': 0} +step=42600 lr=6.000000e-04 loss=2.6239 ce=2.6239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1930014807148837e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006335524958558381, 'last_eos_top1': 0} +step=42650 lr=6.000000e-04 loss=2.7388 ce=2.7388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.033756143442588e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011342258658260107, 'last_eos_top1': 0} +step=42700 lr=6.000000e-04 loss=2.3277 ce=2.3277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0882446076720953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001305047539062798, 'last_eos_top1': 0} +step=42750 lr=6.000000e-04 loss=2.2479 ce=2.2479 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6824984161066823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031227576546370983, 'last_eos_top1': 0} +step=42800 lr=6.000000e-04 loss=2.4052 ce=2.4052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.843953997304197e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018848518375307322, 'last_eos_top1': 0} +step=42850 lr=6.000000e-04 loss=2.7429 ce=2.7429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2578422683873214e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016253412468358874, 'last_eos_top1': 0} +step=42900 lr=6.000000e-04 loss=3.6699 ce=3.6699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.848751920391805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012888731434941292, 'last_eos_top1': 0} +step=42950 lr=6.000000e-04 loss=2.4240 ce=2.4240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.100637726078276e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003005050588399172, 'last_eos_top1': 0} +step=43000 lr=6.000000e-04 loss=2.4930 ce=2.4930 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.803254447935615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025363038294017315, 'last_eos_top1': 0} +step=43050 lr=6.000000e-04 loss=2.8851 ce=2.8851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1761359019146767e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002191766630858183, 'last_eos_top1': 0} +step=43100 lr=6.000000e-04 loss=2.5589 ce=2.5589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2914910257677548e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002006880007684231, 'last_eos_top1': 0} +step=43150 lr=6.000000e-04 loss=2.5253 ce=2.5253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.577616058057174e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020690937526524067, 'last_eos_top1': 0} +step=43200 lr=6.000000e-04 loss=2.9889 ce=2.9889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.324435379501665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016272636130452156, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=700000 rows=691809 buffered=8191 part=openwebtext-t5 +step=43250 lr=6.000000e-04 loss=2.5233 ce=2.5233 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6344514380034525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001972511876374483, 'last_eos_top1': 0} +step=43300 lr=6.000000e-04 loss=2.9233 ce=2.9233 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0717338227550499e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002241313923150301, 'last_eos_top1': 0} +step=43350 lr=6.000000e-04 loss=2.8045 ce=2.8045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.809284514019964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000760322087444365, 'last_eos_top1': 0} +step=43400 lr=6.000000e-04 loss=3.2386 ce=3.2386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.834455002215691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008892402984201908, 'last_eos_top1': 0} +step=43450 lr=6.000000e-04 loss=2.7342 ce=2.7342 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7677903088042513e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019465051591396332, 'last_eos_top1': 0} +step=43500 lr=6.000000e-04 loss=2.4931 ce=2.4931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6014604625524953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013738514389842749, 'last_eos_top1': 0} +step=43550 lr=6.000000e-04 loss=2.0034 ce=2.0034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0714437848946545e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030800052918493748, 'last_eos_top1': 0} +step=43600 lr=6.000000e-04 loss=2.7640 ce=2.7640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2096164937247522e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030036158859729767, 'last_eos_top1': 0} +step=43650 lr=6.000000e-04 loss=2.8754 ce=2.8754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.890387445921078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019345104228705168, 'last_eos_top1': 0} +step=43700 lr=6.000000e-04 loss=2.5220 ce=2.5220 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.467583751946222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032011813018471003, 'last_eos_top1': 0} +step=43750 lr=6.000000e-04 loss=3.0208 ce=3.0208 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.087172859115526e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023443757090717554, 'last_eos_top1': 0} +step=43800 lr=6.000000e-04 loss=2.4631 ce=2.4631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5107720173546113e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001869257539510727, 'last_eos_top1': 0} +step=43850 lr=6.000000e-04 loss=2.3313 ce=2.3313 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.347687980858609e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001732710050418973, 'last_eos_top1': 0} +step=43900 lr=6.000000e-04 loss=2.5543 ce=2.5543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.694565980869811e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015603210777044296, 'last_eos_top1': 0} +step=43950 lr=6.000000e-04 loss=2.5763 ce=2.5763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4770810594200157e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002752126310952008, 'last_eos_top1': 0} +step=44000 lr=6.000000e-04 loss=2.2534 ce=2.2534 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.225252658012323e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024511166848242283, 'last_eos_top1': 0} +step=44050 lr=6.000000e-04 loss=2.8434 ce=2.8434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.81601080839755e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017084795981645584, 'last_eos_top1': 0} +step=44100 lr=6.000000e-04 loss=2.1679 ce=2.1679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3031378077575937e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032198242843151093, 'last_eos_top1': 0} +step=44150 lr=6.000000e-04 loss=2.5103 ce=2.5103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2691510164586361e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003099648281931877, 'last_eos_top1': 0} +step=44200 lr=6.000000e-04 loss=2.5296 ce=2.5296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.866485248086974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024991456884890795, 'last_eos_top1': 0} +step=44250 lr=6.000000e-04 loss=2.8222 ce=2.8222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6198220691876486e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025848159566521645, 'last_eos_top1': 0} +step=44300 lr=6.000000e-04 loss=2.5109 ce=2.5109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3212062185630202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016410921234637499, 'last_eos_top1': 0} +step=44350 lr=6.000000e-04 loss=2.4205 ce=2.4205 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8005135643761605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038087195716798306, 'last_eos_top1': 0} +step=44400 lr=6.000000e-04 loss=2.2286 ce=2.2286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6657109881634824e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032029058784246445, 'last_eos_top1': 0} +step=44450 lr=6.000000e-04 loss=2.7703 ce=2.7703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1962438293267041e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001495129312388599, 'last_eos_top1': 0} +step=44500 lr=6.000000e-04 loss=2.3456 ce=2.3456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2542692275019363e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002676647622138262, 'last_eos_top1': 0} +step=44550 lr=6.000000e-04 loss=2.6943 ce=2.6943 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.881019852589816e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038681982550770044, 'last_eos_top1': 0} +step=44600 lr=6.000000e-04 loss=2.9254 ce=2.9254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6551748558413237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031140900682657957, 'last_eos_top1': 0} +step=44650 lr=6.000000e-04 loss=2.7557 ce=2.7557 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5918460121611133e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001465765293687582, 'last_eos_top1': 0} +step=44700 lr=6.000000e-04 loss=2.3951 ce=2.3951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1664936866727658e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007392188999801874, 'last_eos_top1': 0} +step=44750 lr=6.000000e-04 loss=2.3370 ce=2.3370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4595389075111598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00210035452619195, 'last_eos_top1': 0} +step=44800 lr=6.000000e-04 loss=2.7993 ce=2.7993 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1879583325935528e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003361170180141926, 'last_eos_top1': 0} +step=44850 lr=6.000000e-04 loss=3.1983 ce=3.1983 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2285583579796366e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020516770891845226, 'last_eos_top1': 0} +step=44900 lr=6.000000e-04 loss=2.1537 ce=2.1537 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.200665701617254e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003100417088717222, 'last_eos_top1': 0} +step=44950 lr=6.000000e-04 loss=2.5126 ce=2.5126 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.847156434261706e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032590501941740513, 'last_eos_top1': 0} +step=45000 lr=6.000000e-04 loss=2.6369 ce=2.6369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.025265636504628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001723828841932118, 'last_eos_top1': 0} +step=45050 lr=6.000000e-04 loss=2.7060 ce=2.7060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.675920642213896e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010681403800845146, 'last_eos_top1': 0} +step=45100 lr=6.000000e-04 loss=2.7482 ce=2.7482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3766113625024445e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008302556816488504, 'last_eos_top1': 0} +step=45150 lr=6.000000e-04 loss=2.5474 ce=2.5474 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.103579602902755e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000605134991928935, 'last_eos_top1': 0} +step=45200 lr=6.000000e-04 loss=2.6195 ce=2.6195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4426639609155245e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009076162823475897, 'last_eos_top1': 0} +step=45250 lr=6.000000e-04 loss=2.8615 ce=2.8615 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.040713065478485e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018028633203357458, 'last_eos_top1': 0} +step=45300 lr=6.000000e-04 loss=2.2610 ce=2.2610 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.178071195492521e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002514890395104885, 'last_eos_top1': 0} +step=45350 lr=6.000000e-04 loss=2.8871 ce=2.8871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1514741345308721e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012701796367764473, 'last_eos_top1': 0} +step=45400 lr=6.000000e-04 loss=2.8905 ce=2.8905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5752153305802494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022973278537392616, 'last_eos_top1': 0} +step=45450 lr=6.000000e-04 loss=2.3431 ce=2.3431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.380308615101967e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033005629666149616, 'last_eos_top1': 0} +step=45500 lr=6.000000e-04 loss=3.0904 ce=3.0904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.189972292399034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036761946976184845, 'last_eos_top1': 0} +step=45550 lr=6.000000e-04 loss=2.7003 ce=2.7003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.290009640797507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028493425343185663, 'last_eos_top1': 0} +step=45600 lr=6.000000e-04 loss=2.7735 ce=2.7735 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.947184480552096e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005251649301499128, 'last_eos_top1': 0} +step=45650 lr=6.000000e-04 loss=2.2401 ce=2.2401 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.825213858974166e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003956334199756384, 'last_eos_top1': 0} +step=45700 lr=6.000000e-04 loss=3.3078 ce=3.3078 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6450412658741698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003564517479389906, 'last_eos_top1': 0} +step=45750 lr=6.000000e-04 loss=2.5617 ce=2.5617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.323275672708405e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005564388819038868, 'last_eos_top1': 0} +step=45800 lr=6.000000e-04 loss=2.6067 ce=2.6067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9006391084985808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005927725695073605, 'last_eos_top1': 0} +step=45850 lr=6.000000e-04 loss=2.8441 ce=2.8441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3023006431467365e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005442294292151928, 'last_eos_top1': 0} +step=45900 lr=6.000000e-04 loss=2.5057 ce=2.5057 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2570004400913604e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006249048747122288, 'last_eos_top1': 0} +step=45950 lr=6.000000e-04 loss=2.7988 ce=2.7988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.268368557968643e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006537993438541889, 'last_eos_top1': 0} +step=46000 lr=6.000000e-04 loss=2.2058 ce=2.2058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0622445188346319e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006863420829176903, 'last_eos_top1': 0} +step=46050 lr=6.000000e-04 loss=2.1519 ce=2.1519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5444864402525127e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006971991620957851, 'last_eos_top1': 0} +step=46100 lr=6.000000e-04 loss=2.7827 ce=2.7827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.703209459781647e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00724275317043066, 'last_eos_top1': 0} +step=46150 lr=6.000000e-04 loss=3.0005 ce=3.0005 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5532323232037015e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005707111209630966, 'last_eos_top1': 0} +step=46200 lr=6.000000e-04 loss=2.7115 ce=2.7115 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1109499104786664e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004425969906151295, 'last_eos_top1': 0} +step=46250 lr=6.000000e-04 loss=2.6887 ce=2.6887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2790945220331196e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00393515732139349, 'last_eos_top1': 0} +step=46300 lr=6.000000e-04 loss=2.4349 ce=2.4349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.25876122689806e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005308182910084724, 'last_eos_top1': 0} +step=46350 lr=6.000000e-04 loss=2.5556 ce=2.5556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4424253095057793e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006087789312005043, 'last_eos_top1': 0} +step=46400 lr=6.000000e-04 loss=2.7308 ce=2.7308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.409262742730789e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006275189109146595, 'last_eos_top1': 0} +step=46450 lr=6.000000e-04 loss=2.5391 ce=2.5391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2195330529939383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004305632784962654, 'last_eos_top1': 0} +step=46500 lr=6.000000e-04 loss=2.4662 ce=2.4662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.448609782528365e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016699173720553517, 'last_eos_top1': 0} +step=46550 lr=6.000000e-04 loss=2.5351 ce=2.5351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1889629351790063e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026835775934159756, 'last_eos_top1': 0} +step=46600 lr=6.000000e-04 loss=3.0058 ce=3.0058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.04824136744719e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023987763561308384, 'last_eos_top1': 0} +step=46650 lr=6.000000e-04 loss=2.8366 ce=2.8366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.962625204527285e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013014735886827111, 'last_eos_top1': 0} +step=46700 lr=6.000000e-04 loss=2.3434 ce=2.3434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.198530849767849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032040507066994905, 'last_eos_top1': 0} +step=46750 lr=6.000000e-04 loss=2.6241 ce=2.6241 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1010357411578298e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038867094554007053, 'last_eos_top1': 0} +step=46800 lr=6.000000e-04 loss=2.6441 ce=2.6441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.013521796267014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032595552038401365, 'last_eos_top1': 0} +step=46850 lr=6.000000e-04 loss=2.4572 ce=2.4572 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.655561264778953e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034614522010087967, 'last_eos_top1': 0} +step=46900 lr=6.000000e-04 loss=2.9482 ce=2.9482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.798476301308256e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005982175003737211, 'last_eos_top1': 0} +step=46950 lr=6.000000e-04 loss=2.6833 ce=2.6833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.446892363484949e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004621739499270916, 'last_eos_top1': 0} +step=47000 lr=6.000000e-04 loss=2.9733 ce=2.9733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3364799997361843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036553358659148216, 'last_eos_top1': 0} +step=47050 lr=6.000000e-04 loss=2.7539 ce=2.7539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.139266441896325e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028854585252702236, 'last_eos_top1': 0} +step=47100 lr=6.000000e-04 loss=2.5480 ce=2.5480 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0609570381348021e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011599789140745997, 'last_eos_top1': 0} +step=47150 lr=6.000000e-04 loss=2.8086 ce=2.8086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.097901960951276e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001368554076179862, 'last_eos_top1': 0} +step=47200 lr=6.000000e-04 loss=2.3274 ce=2.3274 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4568206097465008e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00231408653780818, 'last_eos_top1': 0} +step=47250 lr=6.000000e-04 loss=2.6509 ce=2.6509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3051947462372482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020610347855836153, 'last_eos_top1': 0} +step=47300 lr=6.000000e-04 loss=2.8851 ce=2.8851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1590285541606136e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028594746254384518, 'last_eos_top1': 0} +step=47350 lr=6.000000e-04 loss=2.1303 ce=2.1303 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.239939567516558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017992621287703514, 'last_eos_top1': 0} +step=47400 lr=6.000000e-04 loss=2.6791 ce=2.6791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.73744045040803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001960951369255781, 'last_eos_top1': 0} +step=47450 lr=6.000000e-04 loss=2.2284 ce=2.2284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8443261069478467e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018840630073100328, 'last_eos_top1': 0} +step=47500 lr=6.000000e-04 loss=2.2425 ce=2.2425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6830199456308037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019002060871571302, 'last_eos_top1': 0} +step=47550 lr=6.000000e-04 loss=2.4469 ce=2.4469 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.883340342435986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002521780552342534, 'last_eos_top1': 0} +step=47600 lr=6.000000e-04 loss=2.3814 ce=2.3814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.47297900589183e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003950960002839565, 'last_eos_top1': 0} +step=47650 lr=6.000000e-04 loss=2.5880 ce=2.5880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4102335626375861e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003633948741480708, 'last_eos_top1': 0} +step=47700 lr=6.000000e-04 loss=2.9491 ce=2.9491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2150398106314242e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024171005934476852, 'last_eos_top1': 0} +step=47750 lr=6.000000e-04 loss=3.0465 ce=3.0465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.664661795075517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017068409360945225, 'last_eos_top1': 0} +step=47800 lr=6.000000e-04 loss=2.8360 ce=2.8360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2372486708045471e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023635965771973133, 'last_eos_top1': 0} +step=47850 lr=6.000000e-04 loss=1.9609 ce=1.9609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.223985211458057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019374305848032236, 'last_eos_top1': 0} +step=47900 lr=6.000000e-04 loss=2.7278 ce=2.7278 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3442484487313777e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002373172203078866, 'last_eos_top1': 0} +step=47950 lr=6.000000e-04 loss=2.2476 ce=2.2476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5463951058336534e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003379559377208352, 'last_eos_top1': 0} +step=48000 lr=6.000000e-04 loss=2.5799 ce=2.5799 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0698344087577425e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003092495258897543, 'last_eos_top1': 0} +step=48050 lr=6.000000e-04 loss=2.5001 ce=2.5001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.64996763993986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002446234691888094, 'last_eos_top1': 0} +step=48100 lr=6.000000e-04 loss=3.2435 ce=3.2435 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.556865026941523e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029595564119517803, 'last_eos_top1': 0} +step=48150 lr=6.000000e-04 loss=2.9267 ce=2.9267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.264472853625193e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026852539740502834, 'last_eos_top1': 0} +step=48200 lr=6.000000e-04 loss=2.8461 ce=2.8461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3737549781799316e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002108745975419879, 'last_eos_top1': 0} +step=48250 lr=6.000000e-04 loss=2.8700 ce=2.8700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.944101864472032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002548269694671035, 'last_eos_top1': 0} +step=48300 lr=6.000000e-04 loss=2.4643 ce=2.4643 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4611326150770765e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002602178603410721, 'last_eos_top1': 0} +step=48350 lr=6.000000e-04 loss=2.6515 ce=2.6515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.183341555588413e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002806362695991993, 'last_eos_top1': 0} +step=48400 lr=6.000000e-04 loss=2.4269 ce=2.4269 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9348177122301422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00313460361212492, 'last_eos_top1': 0} +step=48450 lr=6.000000e-04 loss=3.2389 ce=3.2389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0875410225708038e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022451221011579037, 'last_eos_top1': 0} +step=48500 lr=6.000000e-04 loss=2.3704 ce=2.3704 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4014885891810991e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002531331032514572, 'last_eos_top1': 0} +step=48550 lr=6.000000e-04 loss=2.3288 ce=2.3288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3108088498702273e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004169476218521595, 'last_eos_top1': 0} +step=48600 lr=6.000000e-04 loss=2.7691 ce=2.7691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5173476387863047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005495938938111067, 'last_eos_top1': 0} +step=48650 lr=6.000000e-04 loss=2.2955 ce=2.2955 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6796777345007285e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004157103598117828, 'last_eos_top1': 0} +step=48700 lr=6.000000e-04 loss=2.5461 ce=2.5461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1687781807268038e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0040198322385549545, 'last_eos_top1': 0} +step=48750 lr=6.000000e-04 loss=2.6593 ce=2.6593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.317231160821393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044982717372477055, 'last_eos_top1': 0} +step=48800 lr=6.000000e-04 loss=2.6357 ce=2.6357 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6483005058253184e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043945228680968285, 'last_eos_top1': 0} +step=48850 lr=6.000000e-04 loss=2.5987 ce=2.5987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2752730728825554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005498613230884075, 'last_eos_top1': 0} +step=48900 lr=6.000000e-04 loss=2.3349 ce=2.3349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.685273653303739e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037131120916455984, 'last_eos_top1': 0} +step=48950 lr=6.000000e-04 loss=3.0330 ce=3.0330 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.968118320102803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007161034271121025, 'last_eos_top1': 0} +step=49000 lr=6.000000e-04 loss=1.8573 ce=1.8573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0666586604202166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00764474319294095, 'last_eos_top1': 0} +step=49050 lr=6.000000e-04 loss=2.3438 ce=2.3438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.943147071870044e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00507225701585412, 'last_eos_top1': 0} +step=49100 lr=6.000000e-04 loss=2.3528 ce=2.3528 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8164775610785e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007339589763432741, 'last_eos_top1': 0} +step=49150 lr=6.000000e-04 loss=2.6630 ce=2.6630 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.3506986407446675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004052371717989445, 'last_eos_top1': 0} +step=49200 lr=6.000000e-04 loss=2.6189 ce=2.6189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.661730938300025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034213829785585403, 'last_eos_top1': 0} +step=49250 lr=6.000000e-04 loss=2.7662 ce=2.7662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3844896784576122e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004583972506225109, 'last_eos_top1': 0} +step=49300 lr=6.000000e-04 loss=3.2160 ce=3.2160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4481974833179265e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003392287530004978, 'last_eos_top1': 0} +step=49350 lr=6.000000e-04 loss=2.6621 ce=2.6621 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.216343642037828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002819583285599947, 'last_eos_top1': 0} +step=49400 lr=6.000000e-04 loss=2.3455 ce=2.3455 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.560891829896718e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034708702005445957, 'last_eos_top1': 0} +step=49450 lr=6.000000e-04 loss=2.3473 ce=2.3473 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3413078704616055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032850955612957478, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=5 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=800000 rows=791809 buffered=8191 part=openwebtext-t5 +step=49500 lr=6.000000e-04 loss=2.5148 ce=2.5148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6655572835588828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004403169732540846, 'last_eos_top1': 0} +step=49550 lr=6.000000e-04 loss=2.6122 ce=2.6122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4490124158328399e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0051977140828967094, 'last_eos_top1': 0} +step=49600 lr=6.000000e-04 loss=2.8223 ce=2.8223 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3704730008612387e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004108879715204239, 'last_eos_top1': 0} +step=49650 lr=6.000000e-04 loss=2.5180 ce=2.5180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8657232178375125e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033028796315193176, 'last_eos_top1': 0} +step=49700 lr=6.000000e-04 loss=2.4586 ce=2.4586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5942308891681023e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0053611379116773605, 'last_eos_top1': 0} +step=49750 lr=6.000000e-04 loss=2.4781 ce=2.4781 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0474615010025445e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054436009377241135, 'last_eos_top1': 0} +step=49800 lr=6.000000e-04 loss=2.7203 ce=2.7203 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.294290066463873e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0055945077911019325, 'last_eos_top1': 0} +step=49850 lr=6.000000e-04 loss=2.7706 ce=2.7706 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4023757103132084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005038550589233637, 'last_eos_top1': 0} +step=49900 lr=6.000000e-04 loss=2.5820 ce=2.5820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.773402103339322e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035975095815956593, 'last_eos_top1': 0} +step=49950 lr=6.000000e-04 loss=2.4899 ce=2.4899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.700389116711449e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004793460015207529, 'last_eos_top1': 0} +step=50000 lr=6.000000e-04 loss=2.1284 ce=2.1284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.096909707295708e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013841618783771992, 'last_eos_top1': 0} +step=50050 lr=6.000000e-04 loss=2.6289 ce=2.6289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.732333309220849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030027846805751324, 'last_eos_top1': 0} +step=50100 lr=6.000000e-04 loss=2.9782 ce=2.9782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.092134521575645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004305307753384113, 'last_eos_top1': 0} +step=50150 lr=6.000000e-04 loss=3.0621 ce=3.0621 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.253333701053634e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004233729559928179, 'last_eos_top1': 0} +step=50200 lr=6.000000e-04 loss=2.4522 ce=2.4522 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.109007932129316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023693935945630074, 'last_eos_top1': 0} +step=50250 lr=6.000000e-04 loss=2.3235 ce=2.3235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.153221981250681e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037760729901492596, 'last_eos_top1': 0} +step=50300 lr=6.000000e-04 loss=2.4261 ce=2.4261 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.996078587282682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003639079164713621, 'last_eos_top1': 0} +step=50350 lr=6.000000e-04 loss=3.0111 ce=3.0111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.688050355776795e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004905605688691139, 'last_eos_top1': 0} +step=50400 lr=6.000000e-04 loss=2.2196 ce=2.2196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4859706425340846e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005047618877142668, 'last_eos_top1': 0} +step=50450 lr=6.000000e-04 loss=2.5399 ce=2.5399 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.485824087576475e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004387576598674059, 'last_eos_top1': 0} +step=50500 lr=6.000000e-04 loss=2.1377 ce=2.1377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.277487849118188e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00387636199593544, 'last_eos_top1': 0} +step=50550 lr=6.000000e-04 loss=2.9700 ce=2.9700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.149398485722486e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004490359220653772, 'last_eos_top1': 0} +step=50600 lr=6.000000e-04 loss=1.9502 ce=1.9502 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7999995179707184e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028037363663315773, 'last_eos_top1': 0} +step=50650 lr=6.000000e-04 loss=2.6431 ce=2.6431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0663602703716606e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034371265210211277, 'last_eos_top1': 0} +step=50700 lr=6.000000e-04 loss=2.8855 ce=2.8855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7942904125666246e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003263189923018217, 'last_eos_top1': 0} +step=50750 lr=6.000000e-04 loss=2.0890 ce=2.0890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.003568053827621e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030212774872779846, 'last_eos_top1': 0} +step=50800 lr=6.000000e-04 loss=2.4788 ce=2.4788 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1827116395579651e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003666417673230171, 'last_eos_top1': 0} +step=50850 lr=6.000000e-04 loss=2.4107 ce=2.4107 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.205727832915727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003400426125153899, 'last_eos_top1': 0} +step=50900 lr=6.000000e-04 loss=2.7890 ce=2.7890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1704249118338339e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002211096230894327, 'last_eos_top1': 0} +step=50950 lr=6.000000e-04 loss=2.6441 ce=2.6441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2080816052039154e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003134932368993759, 'last_eos_top1': 0} +step=51000 lr=6.000000e-04 loss=2.5192 ce=2.5192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1965673365921248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004067484755069017, 'last_eos_top1': 0} +step=51050 lr=6.000000e-04 loss=2.9330 ce=2.9330 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.218399100063834e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00351032055914402, 'last_eos_top1': 0} +step=51100 lr=6.000000e-04 loss=2.6612 ce=2.6612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6655258150422014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004193299449980259, 'last_eos_top1': 0} +step=51150 lr=6.000000e-04 loss=2.5773 ce=2.5773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.235988656844711e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004271818324923515, 'last_eos_top1': 0} +step=51200 lr=6.000000e-04 loss=2.4334 ce=2.4334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0053692676592618e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004064059816300869, 'last_eos_top1': 0} +step=51250 lr=6.000000e-04 loss=2.2320 ce=2.2320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4294865650299471e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005005300045013428, 'last_eos_top1': 0} +step=51300 lr=6.000000e-04 loss=2.2738 ce=2.2738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.54351765156025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004759310279041529, 'last_eos_top1': 0} +step=51350 lr=6.000000e-04 loss=2.6292 ce=2.6292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9548670024960302e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004400646314024925, 'last_eos_top1': 0} +step=51400 lr=6.000000e-04 loss=2.7126 ce=2.7126 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.75535988091724e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003484179964289069, 'last_eos_top1': 0} +step=51450 lr=6.000000e-04 loss=2.5339 ce=2.5339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9629529560916126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004041633568704128, 'last_eos_top1': 0} +step=51500 lr=6.000000e-04 loss=2.5044 ce=2.5044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0406943298876286e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005476957652717829, 'last_eos_top1': 0} +step=51550 lr=6.000000e-04 loss=2.2595 ce=2.2595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6415669961133972e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0047240667045116425, 'last_eos_top1': 0} +step=51600 lr=6.000000e-04 loss=2.6642 ce=2.6642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3767751776613295e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005266798660159111, 'last_eos_top1': 0} +step=51650 lr=6.000000e-04 loss=2.2772 ce=2.2772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3490574423922226e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00568618206307292, 'last_eos_top1': 0} +step=51700 lr=6.000000e-04 loss=2.2496 ce=2.2496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4030563256237656e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004466446116566658, 'last_eos_top1': 0} +step=51750 lr=6.000000e-04 loss=2.1103 ce=2.1103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7942185624851845e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006263730116188526, 'last_eos_top1': 0} +step=51800 lr=6.000000e-04 loss=3.0083 ce=3.0083 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0502123991027474e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007328635081648827, 'last_eos_top1': 0} +step=51850 lr=6.000000e-04 loss=2.5842 ce=2.5842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8763330192305148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00504656508564949, 'last_eos_top1': 0} +step=51900 lr=6.000000e-04 loss=2.8310 ce=2.8310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.733114812755957e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004985773470252752, 'last_eos_top1': 0} +step=51950 lr=6.000000e-04 loss=2.5395 ce=2.5395 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.473513788776472e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006118619814515114, 'last_eos_top1': 0} +step=52000 lr=6.000000e-04 loss=2.6699 ce=2.6699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6920690541155636e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005765520967543125, 'last_eos_top1': 0} +step=52050 lr=6.000000e-04 loss=2.5127 ce=2.5127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.750662916805595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038063148967921734, 'last_eos_top1': 0} +step=52100 lr=6.000000e-04 loss=2.8731 ce=2.8731 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.772982068359852e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00328870490193367, 'last_eos_top1': 0} +step=52150 lr=6.000000e-04 loss=2.6963 ce=2.6963 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.011696389876306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035488163121044636, 'last_eos_top1': 0} +step=52200 lr=6.000000e-04 loss=2.9684 ce=2.9684 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2346746845869347e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004168634302914143, 'last_eos_top1': 0} +step=52250 lr=6.000000e-04 loss=2.6376 ce=2.6376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3444401449523866e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037457123398780823, 'last_eos_top1': 0} +step=52300 lr=6.000000e-04 loss=2.7982 ce=2.7982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0971168851247057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002957247430458665, 'last_eos_top1': 0} +step=52350 lr=6.000000e-04 loss=2.5359 ce=2.5359 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.697138890856877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0040259649977087975, 'last_eos_top1': 0} +step=52400 lr=6.000000e-04 loss=2.6524 ce=2.6524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7926786313182674e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004171461332589388, 'last_eos_top1': 0} +step=52450 lr=6.000000e-04 loss=2.9970 ce=2.9970 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.728537790768314e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039168065413832664, 'last_eos_top1': 0} +step=52500 lr=6.000000e-04 loss=2.6596 ce=2.6596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1137591172882821e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004305210895836353, 'last_eos_top1': 0} +step=52550 lr=6.000000e-04 loss=2.3280 ce=2.3280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2650739992968738e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043018413707613945, 'last_eos_top1': 0} +step=52600 lr=6.000000e-04 loss=2.5650 ce=2.5650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2709833501721732e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031175054609775543, 'last_eos_top1': 0} +step=52650 lr=6.000000e-04 loss=2.3447 ce=2.3447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.903000495280139e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0041658515110611916, 'last_eos_top1': 0} +step=52700 lr=6.000000e-04 loss=2.7536 ce=2.7536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2830107152694836e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0040549286641180515, 'last_eos_top1': 0} +step=52750 lr=6.000000e-04 loss=2.9271 ce=2.9271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7517195374239236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005525590851902962, 'last_eos_top1': 0} +step=52800 lr=6.000000e-04 loss=2.5545 ce=2.5545 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9835561033687554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005849603097885847, 'last_eos_top1': 0} +step=52850 lr=6.000000e-04 loss=2.8619 ce=2.8619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8126527467975393e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004349389113485813, 'last_eos_top1': 0} +step=52900 lr=6.000000e-04 loss=2.0855 ce=2.0855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.438566298224032e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003982359543442726, 'last_eos_top1': 0} +step=52950 lr=6.000000e-04 loss=2.4073 ce=2.4073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1782580006401986e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003716540988534689, 'last_eos_top1': 0} +step=53000 lr=6.000000e-04 loss=2.5963 ce=2.5963 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4668377843918279e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003986562602221966, 'last_eos_top1': 0} +step=53050 lr=6.000000e-04 loss=2.8631 ce=2.8631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0055782442796044e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034561981447041035, 'last_eos_top1': 0} +step=53100 lr=6.000000e-04 loss=2.9915 ce=2.9915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0457224309211597e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003002284560352564, 'last_eos_top1': 0} +step=53150 lr=6.000000e-04 loss=2.1072 ce=2.1072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.259945515310392e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003110087476670742, 'last_eos_top1': 0} +step=53200 lr=6.000000e-04 loss=2.8035 ce=2.8035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7996833776123822e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002361881546676159, 'last_eos_top1': 0} +step=53250 lr=6.000000e-04 loss=2.5059 ce=2.5059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4473341909470037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002481746720150113, 'last_eos_top1': 0} +step=53300 lr=6.000000e-04 loss=2.6015 ce=2.6015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4128052498563193e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002456442452967167, 'last_eos_top1': 0} +step=53350 lr=6.000000e-04 loss=2.8434 ce=2.8434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.122295755834784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031278009992092848, 'last_eos_top1': 0} +step=53400 lr=6.000000e-04 loss=2.8607 ce=2.8607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.818619219004177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003031424479559064, 'last_eos_top1': 0} +step=53450 lr=6.000000e-04 loss=2.5069 ce=2.5069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4404172513168305e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003444073023274541, 'last_eos_top1': 0} +step=53500 lr=6.000000e-04 loss=2.5504 ce=2.5504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5362586054834537e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001168055459856987, 'last_eos_top1': 0} +step=53550 lr=6.000000e-04 loss=2.9227 ce=2.9227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.904926284623798e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003962340764701366, 'last_eos_top1': 0} +step=53600 lr=6.000000e-04 loss=2.8503 ce=2.8503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.85430544661358e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004046271555125713, 'last_eos_top1': 0} +step=53650 lr=6.000000e-04 loss=2.7165 ce=2.7165 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6312110498547554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032439096830785275, 'last_eos_top1': 0} +step=53700 lr=6.000000e-04 loss=2.6854 ce=2.6854 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0044001101050526e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027648932300508022, 'last_eos_top1': 0} +step=53750 lr=6.000000e-04 loss=2.3847 ce=2.3847 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.579924537509214e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036168000660836697, 'last_eos_top1': 0} +step=53800 lr=6.000000e-04 loss=2.5323 ce=2.5323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0967865819111466e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036620015744119883, 'last_eos_top1': 0} +step=53850 lr=6.000000e-04 loss=1.9358 ce=1.9358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5580328181385994e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003777804784476757, 'last_eos_top1': 0} +step=53900 lr=6.000000e-04 loss=2.4910 ce=2.4910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1095728041254915e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002755019348114729, 'last_eos_top1': 0} +step=53950 lr=6.000000e-04 loss=2.4840 ce=2.4840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0665266017895192e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038354452699422836, 'last_eos_top1': 0} +step=54000 lr=6.000000e-04 loss=3.1985 ce=3.1985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.651069608982652e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025658372323960066, 'last_eos_top1': 0} +step=54050 lr=6.000000e-04 loss=2.0381 ce=2.0381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7035192033508793e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001976754516363144, 'last_eos_top1': 0} +step=54100 lr=6.000000e-04 loss=2.6602 ce=2.6602 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0929315067187417e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020709140226244926, 'last_eos_top1': 0} +step=54150 lr=6.000000e-04 loss=2.5956 ce=2.5956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.669001176196616e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031072567217051983, 'last_eos_top1': 0} +step=54200 lr=6.000000e-04 loss=2.5420 ce=2.5420 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2898627574031707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002036019694060087, 'last_eos_top1': 0} +step=54250 lr=6.000000e-04 loss=2.4532 ce=2.4532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0902296708081849e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017490749014541507, 'last_eos_top1': 0} +step=54300 lr=6.000000e-04 loss=2.5176 ce=2.5176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2493339454522356e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001682153670117259, 'last_eos_top1': 0} +step=54350 lr=6.000000e-04 loss=2.7584 ce=2.7584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.577150760567747e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017013876931741834, 'last_eos_top1': 0} +step=54400 lr=6.000000e-04 loss=2.4408 ce=2.4408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.065184344246518e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009117941372096539, 'last_eos_top1': 0} +step=54450 lr=6.000000e-04 loss=2.8697 ce=2.8697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.285114977799822e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.890735287219286e-05, 'last_eos_top1': 0} +step=54500 lr=6.000000e-04 loss=2.5352 ce=2.5352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.993778803618625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001935117325047031, 'last_eos_top1': 0} +step=54550 lr=6.000000e-04 loss=2.8472 ce=2.8472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.153357789007714e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002044722787104547, 'last_eos_top1': 0} +step=54600 lr=6.000000e-04 loss=2.5762 ce=2.5762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7939011740963906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001946612261235714, 'last_eos_top1': 0} +step=54650 lr=6.000000e-04 loss=2.9451 ce=2.9451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.373089435219299e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019121322548016906, 'last_eos_top1': 0} +step=54700 lr=6.000000e-04 loss=2.3701 ce=2.3701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5153269487200305e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004175066947937012, 'last_eos_top1': 0} +step=54750 lr=6.000000e-04 loss=2.6848 ce=2.6848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.736398063367233e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001732735661789775, 'last_eos_top1': 0} +step=54800 lr=6.000000e-04 loss=2.4042 ce=2.4042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.096707799471915e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027102369349449873, 'last_eos_top1': 0} +step=54850 lr=6.000000e-04 loss=2.3282 ce=2.3282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1558722690097056e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020455345511436462, 'last_eos_top1': 0} +step=54900 lr=6.000000e-04 loss=2.5591 ce=2.5591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4103181203827262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003340603783726692, 'last_eos_top1': 0} +step=54950 lr=6.000000e-04 loss=2.6199 ce=2.6199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0803441657335497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014687154907733202, 'last_eos_top1': 0} +step=55000 lr=6.000000e-04 loss=2.8190 ce=2.8190 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2521757525973953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020238803699612617, 'last_eos_top1': 0} +step=55050 lr=6.000000e-04 loss=2.7277 ce=2.7277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1496150768361986e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002638255711644888, 'last_eos_top1': 0} +step=55100 lr=6.000000e-04 loss=3.0290 ce=3.0290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.962285750778392e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026580928824841976, 'last_eos_top1': 0} +step=55150 lr=6.000000e-04 loss=2.6299 ce=2.6299 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0026483727851883e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024149396922439337, 'last_eos_top1': 0} +step=55200 lr=6.000000e-04 loss=2.4879 ce=2.4879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5619962141499855e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025179460644721985, 'last_eos_top1': 0} +step=55250 lr=6.000000e-04 loss=2.7874 ce=2.7874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0891956662817392e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002918606624007225, 'last_eos_top1': 0} +step=55300 lr=6.000000e-04 loss=2.8156 ce=2.8156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2721846360363998e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032178061082959175, 'last_eos_top1': 0} +step=55350 lr=6.000000e-04 loss=3.1875 ce=3.1875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1057909432565793e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00333651015534997, 'last_eos_top1': 0} +step=55400 lr=6.000000e-04 loss=2.1959 ce=2.1959 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8422113498672843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035761515609920025, 'last_eos_top1': 0} +step=55450 lr=6.000000e-04 loss=2.2431 ce=2.2431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8007824110100046e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025917747989296913, 'last_eos_top1': 0} +step=55500 lr=6.000000e-04 loss=2.3224 ce=2.3224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.069591598934494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002495251130312681, 'last_eos_top1': 0} +step=55550 lr=6.000000e-04 loss=2.2887 ce=2.2887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.761342355166562e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002298712497577071, 'last_eos_top1': 0} +step=55600 lr=6.000000e-04 loss=2.1650 ce=2.1650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.637719469727017e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013178745284676552, 'last_eos_top1': 0} +step=55650 lr=6.000000e-04 loss=2.2596 ce=2.2596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3799810151103884e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021938695572316647, 'last_eos_top1': 0} +step=55700 lr=6.000000e-04 loss=2.8340 ce=2.8340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.373272066819482e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018654036102816463, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=900000 rows=891809 buffered=8191 part=openwebtext-t5 +step=55750 lr=6.000000e-04 loss=2.2901 ce=2.2901 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1017102224286646e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014900059904903173, 'last_eos_top1': 0} +step=55800 lr=6.000000e-04 loss=2.6993 ce=2.6993 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5414236258948222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022437237203121185, 'last_eos_top1': 0} +step=55850 lr=6.000000e-04 loss=2.5088 ce=2.5088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.604975295776967e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00206516170874238, 'last_eos_top1': 0} +step=55900 lr=6.000000e-04 loss=2.5592 ce=2.5592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6182300896616653e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002662691520527005, 'last_eos_top1': 0} +step=55950 lr=6.000000e-04 loss=2.8084 ce=2.8084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.728483766783029e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002613459015265107, 'last_eos_top1': 0} +step=56000 lr=6.000000e-04 loss=2.7095 ce=2.7095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0991851013386622e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002645169384777546, 'last_eos_top1': 0} +step=56050 lr=6.000000e-04 loss=2.1187 ce=2.1187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0458269571245182e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003021396230906248, 'last_eos_top1': 0} +step=56100 lr=6.000000e-04 loss=2.4648 ce=2.4648 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3647349735256284e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032858015038073063, 'last_eos_top1': 0} +step=56150 lr=6.000000e-04 loss=2.4054 ce=2.4054 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5134992180683184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018834751099348068, 'last_eos_top1': 0} +step=56200 lr=6.000000e-04 loss=2.6807 ce=2.6807 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1542560969246551e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019343665335327387, 'last_eos_top1': 0} +step=56250 lr=6.000000e-04 loss=2.6872 ce=2.6872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.977531590266153e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012830491177737713, 'last_eos_top1': 0} +step=56300 lr=6.000000e-04 loss=3.2856 ce=3.2856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5186322343652137e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036634099669754505, 'last_eos_top1': 0} +step=56350 lr=6.000000e-04 loss=2.7742 ce=2.7742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4987766664708033e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033306190744042397, 'last_eos_top1': 0} +step=56400 lr=6.000000e-04 loss=2.7152 ce=2.7152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3121560186846182e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002028044778853655, 'last_eos_top1': 0} +step=56450 lr=6.000000e-04 loss=2.2388 ce=2.2388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6493742805323564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035416604951024055, 'last_eos_top1': 0} +step=56500 lr=6.000000e-04 loss=2.5493 ce=2.5493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5865844034124166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004598013591021299, 'last_eos_top1': 0} +step=56550 lr=6.000000e-04 loss=2.2592 ce=2.2592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.556238880264573e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005363155622035265, 'last_eos_top1': 0} +step=56600 lr=6.000000e-04 loss=2.8124 ce=2.8124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5145367291988805e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004853846505284309, 'last_eos_top1': 0} +step=56650 lr=6.000000e-04 loss=2.8683 ce=2.8683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0798863261006773e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005266830325126648, 'last_eos_top1': 0} +step=56700 lr=6.000000e-04 loss=2.6829 ce=2.6829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8089142031385563e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004563263151794672, 'last_eos_top1': 0} +step=56750 lr=6.000000e-04 loss=2.9033 ce=2.9033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6472167519386858e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004259740002453327, 'last_eos_top1': 0} +step=56800 lr=6.000000e-04 loss=2.5449 ce=2.5449 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.259464574512094e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027039702981710434, 'last_eos_top1': 0} +step=56850 lr=6.000000e-04 loss=2.7053 ce=2.7053 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.910875238943845e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027407477609813213, 'last_eos_top1': 0} +step=56900 lr=6.000000e-04 loss=2.3113 ce=2.3113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7410631446400657e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002007143571972847, 'last_eos_top1': 0} +step=56950 lr=6.000000e-04 loss=2.4671 ce=2.4671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7751643099472858e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018855941016227007, 'last_eos_top1': 0} +step=57000 lr=6.000000e-04 loss=2.5266 ce=2.5266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.058909351238981e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001256801886484027, 'last_eos_top1': 0} +step=57050 lr=6.000000e-04 loss=2.9137 ce=2.9137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.724914884311147e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018527866341173649, 'last_eos_top1': 0} +step=57100 lr=6.000000e-04 loss=2.5427 ce=2.5427 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.694614861160517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011294451542198658, 'last_eos_top1': 0} +step=57150 lr=6.000000e-04 loss=2.7226 ce=2.7226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4535147784044966e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009656168986111879, 'last_eos_top1': 0} +step=57200 lr=6.000000e-04 loss=2.3323 ce=2.3323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0741097614518367e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012504593469202518, 'last_eos_top1': 0} +step=57250 lr=6.000000e-04 loss=2.3719 ce=2.3719 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.05032920348458e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015516092535108328, 'last_eos_top1': 0} +step=57300 lr=6.000000e-04 loss=2.7760 ce=2.7760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2421636711223982e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011695742141455412, 'last_eos_top1': 0} +step=57350 lr=6.000000e-04 loss=2.7205 ce=2.7205 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.583178325556219e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009110571700148284, 'last_eos_top1': 0} +step=57400 lr=6.000000e-04 loss=2.6418 ce=2.6418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4340384698007256e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014370165299624205, 'last_eos_top1': 0} +step=57450 lr=6.000000e-04 loss=2.5799 ce=2.5799 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8421826098347083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002598099410533905, 'last_eos_top1': 0} +step=57500 lr=6.000000e-04 loss=2.1237 ce=2.1237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.636432539788075e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020024357363581657, 'last_eos_top1': 0} +step=57550 lr=6.000000e-04 loss=2.3868 ce=2.3868 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.704442396061495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017960406839847565, 'last_eos_top1': 0} +step=57600 lr=6.000000e-04 loss=2.4432 ce=2.4432 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2011041690129787e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00169439020100981, 'last_eos_top1': 0} +step=57650 lr=6.000000e-04 loss=2.4955 ce=2.4955 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.30087072547758e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008525930461473763, 'last_eos_top1': 0} +step=57700 lr=6.000000e-04 loss=2.6160 ce=2.6160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5868412447161973e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013620896497741342, 'last_eos_top1': 0} +step=57750 lr=6.000000e-04 loss=2.5699 ce=2.5699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.708115087240003e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015826813178136945, 'last_eos_top1': 0} +step=57800 lr=6.000000e-04 loss=2.0012 ce=2.0012 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3549428331316449e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018578818999230862, 'last_eos_top1': 0} +step=57850 lr=6.000000e-04 loss=2.3962 ce=2.3962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.516104910289869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025268499739468098, 'last_eos_top1': 0} +step=57900 lr=6.000000e-04 loss=3.0629 ce=3.0629 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.077362847165205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002163939643651247, 'last_eos_top1': 0} +step=57950 lr=6.000000e-04 loss=2.3752 ce=2.3752 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2936375899007544e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011529431212693453, 'last_eos_top1': 0} +step=58000 lr=6.000000e-04 loss=2.0075 ce=2.0075 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0651251614035573e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003319282550364733, 'last_eos_top1': 0} +step=58050 lr=6.000000e-04 loss=2.4786 ce=2.4786 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0988909707521088e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008144901366904378, 'last_eos_top1': 0} +step=58100 lr=6.000000e-04 loss=2.5153 ce=2.5153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.3024609668646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007370698731392622, 'last_eos_top1': 0} +step=58150 lr=6.000000e-04 loss=2.4741 ce=2.4741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.528862958366517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010316127445548773, 'last_eos_top1': 0} +step=58200 lr=6.000000e-04 loss=2.2946 ce=2.2946 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3325264919549227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005802265950478613, 'last_eos_top1': 0} +step=58250 lr=6.000000e-04 loss=2.4555 ce=2.4555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.14134034246672e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003864829195663333, 'last_eos_top1': 0} +step=58300 lr=6.000000e-04 loss=2.3743 ce=2.3743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4786022549960762e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001485791290178895, 'last_eos_top1': 0} +step=58350 lr=6.000000e-04 loss=2.7092 ce=2.7092 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3794459846394602e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011506247101351619, 'last_eos_top1': 0} +step=58400 lr=6.000000e-04 loss=2.1997 ce=2.1997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.829485523165204e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015290232840925455, 'last_eos_top1': 0} +step=58450 lr=6.000000e-04 loss=2.2572 ce=2.2572 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3775957743055187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002521518850699067, 'last_eos_top1': 0} +step=58500 lr=6.000000e-04 loss=2.6277 ce=2.6277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9921706300228834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023164846934378147, 'last_eos_top1': 0} +step=58550 lr=6.000000e-04 loss=2.6336 ce=2.6336 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8298458346398547e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001054577762261033, 'last_eos_top1': 0} +step=58600 lr=6.000000e-04 loss=2.5716 ce=2.5716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9222650482552126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00190988858230412, 'last_eos_top1': 0} +step=58650 lr=6.000000e-04 loss=2.7590 ce=2.7590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0993083025095984e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026874011382460594, 'last_eos_top1': 0} +step=58700 lr=6.000000e-04 loss=2.5462 ce=2.5462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.189286745619029e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020649642683565617, 'last_eos_top1': 0} +step=58750 lr=6.000000e-04 loss=2.6770 ce=2.6770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.658050314290449e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001446023816242814, 'last_eos_top1': 0} +step=58800 lr=6.000000e-04 loss=2.2697 ce=2.2697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7244088667212054e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016528599662706256, 'last_eos_top1': 0} +step=58850 lr=6.000000e-04 loss=2.6611 ce=2.6611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.660071029618848e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000700603355653584, 'last_eos_top1': 0} +step=58900 lr=6.000000e-04 loss=2.0391 ce=2.0391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.832910675148014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009897137060761452, 'last_eos_top1': 0} +step=58950 lr=6.000000e-04 loss=2.4390 ce=2.4390 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.214183950854931e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011528157629072666, 'last_eos_top1': 0} +step=59000 lr=6.000000e-04 loss=2.2858 ce=2.2858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4144716260489076e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009568585082888603, 'last_eos_top1': 0} +step=59050 lr=6.000000e-04 loss=2.6007 ce=2.6007 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.244402756128693e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006927524227648973, 'last_eos_top1': 0} +step=59100 lr=6.000000e-04 loss=2.8108 ce=2.8108 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2517039067461155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009249855065718293, 'last_eos_top1': 0} +step=59150 lr=6.000000e-04 loss=2.6339 ce=2.6339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.984593826724449e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021353201009333134, 'last_eos_top1': 0} +step=59200 lr=6.000000e-04 loss=2.9113 ce=2.9113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.406018994515762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002136767143383622, 'last_eos_top1': 0} +step=59250 lr=6.000000e-04 loss=2.2889 ce=2.2889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.418415068765171e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013258135877549648, 'last_eos_top1': 0} +step=59300 lr=6.000000e-04 loss=2.2180 ce=2.2180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5861631254665554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010367805371060967, 'last_eos_top1': 0} +step=59350 lr=6.000000e-04 loss=2.4155 ce=2.4155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.961338759632781e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001445729285478592, 'last_eos_top1': 0} +step=59400 lr=6.000000e-04 loss=2.7266 ce=2.7266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1297045602987055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016621199902147055, 'last_eos_top1': 0} +step=59450 lr=6.000000e-04 loss=1.9388 ce=1.9388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.595080721017439e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007684821030125022, 'last_eos_top1': 0} +step=59500 lr=6.000000e-04 loss=2.1774 ce=2.1774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.619611435278784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000750287901610136, 'last_eos_top1': 0} +step=59550 lr=6.000000e-04 loss=2.5095 ce=2.5095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2363753891550004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014486097497865558, 'last_eos_top1': 0} +step=59600 lr=6.000000e-04 loss=2.5588 ce=2.5588 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.18635271571111e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011642645113170147, 'last_eos_top1': 0} +step=59650 lr=6.000000e-04 loss=2.6021 ce=2.6021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.719868335698266e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00156689016148448, 'last_eos_top1': 0} +step=59700 lr=6.000000e-04 loss=2.1833 ce=2.1833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.771674179617548e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00298504251986742, 'last_eos_top1': 0} +step=59750 lr=6.000000e-04 loss=2.3011 ce=2.3011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.654207704239525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002964687068015337, 'last_eos_top1': 0} +step=59800 lr=6.000000e-04 loss=3.3918 ce=3.3918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4101213057292625e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026163614820688963, 'last_eos_top1': 0} +step=59850 lr=6.000000e-04 loss=2.9254 ce=2.9254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.846376108005643e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036428398452699184, 'last_eos_top1': 0} +step=59900 lr=6.000000e-04 loss=2.6140 ce=2.6140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.541373972548172e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020478982478380203, 'last_eos_top1': 0} +step=59950 lr=6.000000e-04 loss=2.3771 ce=2.3771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.95993253935012e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004931241273880005, 'last_eos_top1': 0} +step=60000 lr=6.000000e-04 loss=2.5752 ce=2.5752 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1158317647641525e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004417705349624157, 'last_eos_top1': 0} +step=60050 lr=6.000000e-04 loss=2.6827 ce=2.6827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7484748241258785e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039231982082128525, 'last_eos_top1': 0} +step=60100 lr=6.000000e-04 loss=2.2375 ce=2.2375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.594351851963438e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002100527286529541, 'last_eos_top1': 0} +step=60150 lr=6.000000e-04 loss=2.7625 ce=2.7625 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6539541067904793e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000331405783072114, 'last_eos_top1': 0} +step=60200 lr=6.000000e-04 loss=2.8418 ce=2.8418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.874531315290369e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00165827467571944, 'last_eos_top1': 0} +step=60250 lr=6.000000e-04 loss=2.9019 ce=2.9019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6250636690529063e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003512706607580185, 'last_eos_top1': 0} +step=60300 lr=6.000000e-04 loss=3.0014 ce=3.0014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9951465219492093e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001046216581016779, 'last_eos_top1': 0} +step=60350 lr=6.000000e-04 loss=2.1515 ce=2.1515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8434446246828884e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002195242792367935, 'last_eos_top1': 0} +step=60400 lr=6.000000e-04 loss=2.3116 ce=2.3116 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.055088018300012e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028775050304830074, 'last_eos_top1': 0} +step=60450 lr=6.000000e-04 loss=2.7204 ce=2.7204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.030821840686258e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005644559860229492, 'last_eos_top1': 0} +step=60500 lr=6.000000e-04 loss=2.7573 ce=2.7573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8968243239214644e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009961979230865836, 'last_eos_top1': 0} +step=60550 lr=6.000000e-04 loss=2.4352 ce=2.4352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6838461053557694e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016643698327243328, 'last_eos_top1': 0} +step=60600 lr=6.000000e-04 loss=2.9407 ce=2.9407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.616311005316675e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021680770441889763, 'last_eos_top1': 0} +step=60650 lr=6.000000e-04 loss=2.5152 ce=2.5152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3775586669216864e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001002827426418662, 'last_eos_top1': 0} +step=60700 lr=6.000000e-04 loss=2.6578 ce=2.6578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5760204405523837e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00163982470985502, 'last_eos_top1': 0} +step=60750 lr=6.000000e-04 loss=2.8212 ce=2.8212 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.947278724401258e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003453300450928509, 'last_eos_top1': 0} +step=60800 lr=6.000000e-04 loss=2.6001 ce=2.6001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6770587535575032e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009281622478738427, 'last_eos_top1': 0} +step=60850 lr=6.000000e-04 loss=3.1448 ce=3.1448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1221824024687521e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000792111037299037, 'last_eos_top1': 0} +step=60900 lr=6.000000e-04 loss=2.3567 ce=2.3567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.612317282706499e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002758956979960203, 'last_eos_top1': 0} +step=60950 lr=6.000000e-04 loss=3.0436 ce=3.0436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.198675575637026e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015170939732342958, 'last_eos_top1': 0} +step=61000 lr=6.000000e-04 loss=2.2801 ce=2.2801 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0466911589901429e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000984148122370243, 'last_eos_top1': 0} +step=61050 lr=6.000000e-04 loss=2.8931 ce=2.8931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.645060345064849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014485635329037905, 'last_eos_top1': 0} +step=61100 lr=6.000000e-04 loss=2.2501 ce=2.2501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0404724889667705e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001206698128953576, 'last_eos_top1': 0} +step=61150 lr=6.000000e-04 loss=2.8019 ce=2.8019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.086867218371481e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001423240639269352, 'last_eos_top1': 0} +step=61200 lr=6.000000e-04 loss=2.6674 ce=2.6674 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.816036794683896e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018219903577119112, 'last_eos_top1': 0} +step=61250 lr=6.000000e-04 loss=2.3804 ce=2.3804 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.017872460186481e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001193176954984665, 'last_eos_top1': 0} +step=61300 lr=6.000000e-04 loss=2.3214 ce=2.3214 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.78456648933934e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023142409045249224, 'last_eos_top1': 0} +step=61350 lr=6.000000e-04 loss=2.4824 ce=2.4824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0127987479791045e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026998789981007576, 'last_eos_top1': 0} +step=61400 lr=6.000000e-04 loss=2.7466 ce=2.7466 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7445161574869417e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015747109428048134, 'last_eos_top1': 0} +step=61450 lr=6.000000e-04 loss=2.9349 ce=2.9349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.36025389819406e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013428156962618232, 'last_eos_top1': 0} +step=61500 lr=6.000000e-04 loss=2.4442 ce=2.4442 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4370636310777627e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020994360966142267, 'last_eos_top1': 0} +step=61550 lr=6.000000e-04 loss=2.9490 ce=2.9490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1384796380298212e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010052576544694602, 'last_eos_top1': 0} +step=61600 lr=6.000000e-04 loss=2.4804 ce=2.4804 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5045290638227016e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001510431175120175, 'last_eos_top1': 0} +step=61650 lr=6.000000e-04 loss=2.4430 ce=2.4430 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.100416274217423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011693781561916694, 'last_eos_top1': 0} +step=61700 lr=6.000000e-04 loss=2.6875 ce=2.6875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.870201104902662e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001295567926717922, 'last_eos_top1': 0} +step=61750 lr=6.000000e-04 loss=2.5049 ce=2.5049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.313094334269408e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011044683924410492, 'last_eos_top1': 0} +step=61800 lr=6.000000e-04 loss=2.3686 ce=2.3686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0020798981713597e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010055743041448295, 'last_eos_top1': 0} +step=61850 lr=6.000000e-04 loss=2.9488 ce=2.9488 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.865868378255982e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016971147852018476, 'last_eos_top1': 0} +step=61900 lr=6.000000e-04 loss=2.4916 ce=2.4916 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.70685835252516e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.159490971593186e-05, 'last_eos_top1': 0} +step=61950 lr=6.000000e-04 loss=2.5598 ce=2.5598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.196998012659606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.967563578858972e-05, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1000000 rows=991809 buffered=8191 part=openwebtext-t5 +step=62000 lr=6.000000e-04 loss=2.8300 ce=2.8300 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.469347343023401e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.483231067657471e-05, 'last_eos_top1': 0} +step=62050 lr=6.000000e-04 loss=2.2723 ce=2.2723 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0500997278722934e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.16038009361364e-05, 'last_eos_top1': 0} +step=62100 lr=6.000000e-04 loss=2.3776 ce=2.3776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.289036147994921e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014309832477010787, 'last_eos_top1': 0} +step=62150 lr=6.000000e-04 loss=2.3440 ce=2.3440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.461277057998814e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.477681017713621e-05, 'last_eos_top1': 0} +step=62200 lr=6.000000e-04 loss=2.9851 ce=2.9851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.748849722906016e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010474210284883156, 'last_eos_top1': 0} +step=62250 lr=6.000000e-04 loss=2.4805 ce=2.4805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.196415703627281e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015206655371002853, 'last_eos_top1': 0} +step=62300 lr=6.000000e-04 loss=2.3397 ce=2.3397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.304662176466081e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002275404112879187, 'last_eos_top1': 0} +step=62350 lr=6.000000e-04 loss=3.0433 ce=3.0433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1945718142669648e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020126707386225462, 'last_eos_top1': 0} +step=62400 lr=6.000000e-04 loss=2.8224 ce=2.8224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.866284820716828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001037887268466875, 'last_eos_top1': 0} +step=62450 lr=6.000000e-04 loss=2.3785 ce=2.3785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.88581723504467e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001563216937938705, 'last_eos_top1': 0} +step=62500 lr=6.000000e-04 loss=3.0041 ce=3.0041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.829703407769557e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002769048442132771, 'last_eos_top1': 0} +step=62550 lr=6.000000e-04 loss=2.6044 ce=2.6044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1034497219952755e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017921041580848396, 'last_eos_top1': 0} +step=62600 lr=6.000000e-04 loss=2.5413 ce=2.5413 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.934531716571655e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.519921200582758e-05, 'last_eos_top1': 0} +step=62650 lr=6.000000e-04 loss=3.0409 ce=3.0409 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.251400115317665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.881698886398226e-05, 'last_eos_top1': 0} +step=62700 lr=6.000000e-04 loss=2.7406 ce=2.7406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.951163075224031e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.193612640956417e-05, 'last_eos_top1': 0} +step=62750 lr=6.000000e-04 loss=2.8587 ce=2.8587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4068435120861977e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040712152258493006, 'last_eos_top1': 0} +step=62800 lr=6.000000e-04 loss=2.3121 ce=2.3121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4077550076763146e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002165258483728394, 'last_eos_top1': 0} +step=62850 lr=6.000000e-04 loss=2.3890 ce=2.3890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2226583748997655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007318275747820735, 'last_eos_top1': 0} +step=62900 lr=6.000000e-04 loss=2.4754 ce=2.4754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.606524493690813e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018464247696101665, 'last_eos_top1': 0} +step=62950 lr=6.000000e-04 loss=1.9551 ce=1.9551 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.84929761721287e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014861447270959616, 'last_eos_top1': 0} +step=63000 lr=6.000000e-04 loss=2.7974 ce=2.7974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.424676030292176e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009526932262815535, 'last_eos_top1': 0} +step=63050 lr=6.000000e-04 loss=2.5315 ce=2.5315 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3597873476101086e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016605958808213472, 'last_eos_top1': 0} +step=63100 lr=6.000000e-04 loss=2.8506 ce=2.8506 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1095491572632454e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007663839496672153, 'last_eos_top1': 0} +step=63150 lr=6.000000e-04 loss=2.9842 ce=2.9842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.104437978938222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024213673896156251, 'last_eos_top1': 0} +step=63200 lr=6.000000e-04 loss=2.8101 ce=2.8101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.864001185633242e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.340349060948938e-05, 'last_eos_top1': 0} +step=63250 lr=6.000000e-04 loss=2.1850 ce=2.1850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1921387340407819e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013741603470407426, 'last_eos_top1': 0} +step=63300 lr=6.000000e-04 loss=2.5325 ce=2.5325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0099219252879266e-05, 'pos0_bos_top1': 0, 'last_eos_p': 9.524109191261232e-05, 'last_eos_top1': 0} +step=63350 lr=6.000000e-04 loss=2.7489 ce=2.7489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2481261364882812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004367296351119876, 'last_eos_top1': 0} +step=63400 lr=6.000000e-04 loss=3.0043 ce=3.0043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.56072609469993e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004338161088526249, 'last_eos_top1': 0} +step=63450 lr=6.000000e-04 loss=2.6248 ce=2.6248 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.037514023977565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006364636938087642, 'last_eos_top1': 0} +step=63500 lr=6.000000e-04 loss=2.7636 ce=2.7636 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3467732969729695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00105567486025393, 'last_eos_top1': 0} +step=63550 lr=6.000000e-04 loss=2.4479 ce=2.4479 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.481168722733855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000893797492608428, 'last_eos_top1': 0} +step=63600 lr=6.000000e-04 loss=2.3336 ce=2.3336 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.544042430003174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010995813645422459, 'last_eos_top1': 0} +step=63650 lr=6.000000e-04 loss=3.3785 ce=3.3785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.298349343414884e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008374451426789165, 'last_eos_top1': 0} +step=63700 lr=6.000000e-04 loss=2.4023 ce=2.4023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3802742614643648e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015912374947220087, 'last_eos_top1': 0} +step=63750 lr=6.000000e-04 loss=2.4467 ce=2.4467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.614558060391573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010040620109066367, 'last_eos_top1': 0} +step=63800 lr=6.000000e-04 loss=3.0276 ce=3.0276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.728064469527453e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001617546658962965, 'last_eos_top1': 0} +step=63850 lr=6.000000e-04 loss=2.3047 ce=2.3047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.1013315644231625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016425545327365398, 'last_eos_top1': 0} +step=63900 lr=6.000000e-04 loss=3.2693 ce=3.2693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0155505151487887e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007390806567855179, 'last_eos_top1': 0} +step=63950 lr=6.000000e-04 loss=2.6151 ce=2.6151 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.211026968434453e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006508249789476395, 'last_eos_top1': 0} +step=64000 lr=6.000000e-04 loss=2.9152 ce=2.9152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.76619432750158e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000973011483438313, 'last_eos_top1': 0} +step=64050 lr=6.000000e-04 loss=2.1791 ce=2.1791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.625537647167221e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016520330682396889, 'last_eos_top1': 0} +step=64100 lr=6.000000e-04 loss=3.5406 ce=3.5406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6570702427998185e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012298845686018467, 'last_eos_top1': 0} +step=64150 lr=6.000000e-04 loss=2.5156 ce=2.5156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.059855978586711e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009290466550737619, 'last_eos_top1': 0} +step=64200 lr=6.000000e-04 loss=2.7284 ce=2.7284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.051234544836916e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004607766168192029, 'last_eos_top1': 0} +step=64250 lr=6.000000e-04 loss=2.5330 ce=2.5330 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4194201867212541e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009235436446033418, 'last_eos_top1': 0} +step=64300 lr=6.000000e-04 loss=1.9026 ce=1.9026 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4028000805410556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005196225829422474, 'last_eos_top1': 0} +step=64350 lr=6.000000e-04 loss=2.2938 ce=2.2938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.893034201813862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.086214074864984e-05, 'last_eos_top1': 0} +step=64400 lr=6.000000e-04 loss=2.5387 ce=2.5387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7362149163964204e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011868275760207325, 'last_eos_top1': 0} +step=64450 lr=6.000000e-04 loss=2.5244 ce=2.5244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.842536898446269e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003014400717802346, 'last_eos_top1': 0} +step=64500 lr=6.000000e-04 loss=1.8756 ce=1.8756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.328656127152499e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006130518740974367, 'last_eos_top1': 0} +step=64550 lr=6.000000e-04 loss=2.4340 ce=2.4340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6933407096075825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047966541023924947, 'last_eos_top1': 0} +step=64600 lr=6.000000e-04 loss=2.8207 ce=2.8207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1557772268133704e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043636930058710277, 'last_eos_top1': 0} +step=64650 lr=6.000000e-04 loss=2.7157 ce=2.7157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.785358659224585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000522542861290276, 'last_eos_top1': 0} +step=64700 lr=6.000000e-04 loss=2.6867 ce=2.6867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.836126102716662e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011451994068920612, 'last_eos_top1': 0} +step=64750 lr=6.000000e-04 loss=2.3517 ce=2.3517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0223966455669142e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013775744009763002, 'last_eos_top1': 0} +step=64800 lr=6.000000e-04 loss=2.3802 ce=2.3802 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0292380466125906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011289480607956648, 'last_eos_top1': 0} +step=64850 lr=6.000000e-04 loss=3.0978 ce=3.0978 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0770427252282389e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021421872079372406, 'last_eos_top1': 0} +step=64900 lr=6.000000e-04 loss=1.6986 ce=1.6986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.402322701935191e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009455439867451787, 'last_eos_top1': 0} +step=64950 lr=6.000000e-04 loss=2.8317 ce=2.8317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.970222213771194e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009027464548125863, 'last_eos_top1': 0} +step=65000 lr=6.000000e-04 loss=3.0686 ce=3.0686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.160943768103607e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018931778613477945, 'last_eos_top1': 0} +step=65050 lr=6.000000e-04 loss=2.1033 ce=2.1033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.216807716758922e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002444082871079445, 'last_eos_top1': 0} +step=65100 lr=6.000000e-04 loss=1.9747 ce=1.9747 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2851877727371175e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020642196759581566, 'last_eos_top1': 0} +step=65150 lr=6.000000e-04 loss=2.0368 ce=2.0368 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0746238331194036e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019975549075752497, 'last_eos_top1': 0} +step=65200 lr=6.000000e-04 loss=2.7883 ce=2.7883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.964999324234668e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029753847047686577, 'last_eos_top1': 0} +step=65250 lr=6.000000e-04 loss=2.4547 ce=2.4547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0518335329834372e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018372738268226385, 'last_eos_top1': 0} +step=65300 lr=6.000000e-04 loss=2.4667 ce=2.4667 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3033189134148415e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029645974282175303, 'last_eos_top1': 0} +step=65350 lr=6.000000e-04 loss=2.6131 ce=2.6131 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0027491953223944e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017112735658884048, 'last_eos_top1': 0} +step=65400 lr=6.000000e-04 loss=2.3624 ce=2.3624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5839239495107904e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027825264260172844, 'last_eos_top1': 0} +step=65450 lr=6.000000e-04 loss=2.0910 ce=2.0910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3979689760599285e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003738255938515067, 'last_eos_top1': 0} +step=65500 lr=6.000000e-04 loss=2.4719 ce=2.4719 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.836006049416028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016534340102225542, 'last_eos_top1': 0} +step=65550 lr=6.000000e-04 loss=2.1242 ce=2.1242 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.551306458073668e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001054225256666541, 'last_eos_top1': 0} +step=65600 lr=6.000000e-04 loss=2.4115 ce=2.4115 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.112116236385191e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017704684287309647, 'last_eos_top1': 0} +step=65650 lr=6.000000e-04 loss=2.0837 ce=2.0837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0247231330140494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016913532745093107, 'last_eos_top1': 0} +step=65700 lr=6.000000e-04 loss=2.3452 ce=2.3452 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.446619408961851e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017261492321267724, 'last_eos_top1': 0} +step=65750 lr=6.000000e-04 loss=2.1145 ce=2.1145 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.174413217901019e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020460067316889763, 'last_eos_top1': 0} +step=65800 lr=6.000000e-04 loss=2.5670 ce=2.5670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.7190421759733e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004008041694760323, 'last_eos_top1': 0} +step=65850 lr=6.000000e-04 loss=2.8156 ce=2.8156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3077802577754483e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019439971074461937, 'last_eos_top1': 0} +step=65900 lr=6.000000e-04 loss=2.4190 ce=2.4190 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.221809634444071e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002641544211655855, 'last_eos_top1': 0} +step=65950 lr=6.000000e-04 loss=2.2263 ce=2.2263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2183473700133618e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001969775650650263, 'last_eos_top1': 0} +step=66000 lr=6.000000e-04 loss=2.5951 ce=2.5951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8172759155277163e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019629867747426033, 'last_eos_top1': 0} +step=66050 lr=6.000000e-04 loss=2.3821 ce=2.3821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.410497578850482e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003038142458535731, 'last_eos_top1': 0} +step=66100 lr=6.000000e-04 loss=2.2070 ce=2.2070 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.083382454060484e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008011831669136882, 'last_eos_top1': 0} +step=66150 lr=6.000000e-04 loss=1.9597 ce=1.9597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.76107151270844e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000828553456813097, 'last_eos_top1': 0} +step=66200 lr=6.000000e-04 loss=2.5110 ce=2.5110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3486564057529904e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006228007841855288, 'last_eos_top1': 0} +step=66250 lr=6.000000e-04 loss=2.7612 ce=2.7612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.562321454519406e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010065245442092419, 'last_eos_top1': 0} +step=66300 lr=6.000000e-04 loss=2.5143 ce=2.5143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.221176590770483e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000297725317068398, 'last_eos_top1': 0} +step=66350 lr=6.000000e-04 loss=3.4289 ce=3.4289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.844974228035426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000980955664999783, 'last_eos_top1': 0} +step=66400 lr=6.000000e-04 loss=2.6341 ce=2.6341 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.543950464925729e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015321297105401754, 'last_eos_top1': 0} +step=66450 lr=6.000000e-04 loss=2.3137 ce=2.3137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0424317224533297e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001342741772532463, 'last_eos_top1': 0} +step=66500 lr=6.000000e-04 loss=2.9242 ce=2.9242 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.782266377238557e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00048545689787715673, 'last_eos_top1': 0} +step=66550 lr=6.000000e-04 loss=2.5408 ce=2.5408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.88729591172887e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013116018380969763, 'last_eos_top1': 0} +step=66600 lr=6.000000e-04 loss=2.9147 ce=2.9147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.279648682218976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001594338333234191, 'last_eos_top1': 0} +step=66650 lr=6.000000e-04 loss=2.9150 ce=2.9150 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.064004188985564e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015048440545797348, 'last_eos_top1': 0} +step=66700 lr=6.000000e-04 loss=2.4166 ce=2.4166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.151122705778107e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017784492811188102, 'last_eos_top1': 0} +step=66750 lr=6.000000e-04 loss=2.7717 ce=2.7717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.057024721580092e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004541605710983276, 'last_eos_top1': 0} +step=66800 lr=6.000000e-04 loss=2.4069 ce=2.4069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2973679733695462e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030794821213930845, 'last_eos_top1': 0} +step=66850 lr=6.000000e-04 loss=2.5292 ce=2.5292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2344708011369221e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019005953799933195, 'last_eos_top1': 0} +step=66900 lr=6.000000e-04 loss=2.6272 ce=2.6272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5270499716280028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043656970374286175, 'last_eos_top1': 0} +step=66950 lr=6.000000e-04 loss=1.8046 ce=1.8046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0954075706831645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0041017550975084305, 'last_eos_top1': 0} +step=67000 lr=6.000000e-04 loss=3.0172 ce=3.0172 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1389854383596685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005491756834089756, 'last_eos_top1': 0} +step=67050 lr=6.000000e-04 loss=2.4726 ce=2.4726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.330843800446019e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003648723941296339, 'last_eos_top1': 0} +step=67100 lr=6.000000e-04 loss=2.3743 ce=2.3743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.7681248765438795e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038729547522962093, 'last_eos_top1': 0} +step=67150 lr=6.000000e-04 loss=2.9069 ce=2.9069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7975402442971244e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00381430983543396, 'last_eos_top1': 0} +step=67200 lr=6.000000e-04 loss=2.0918 ce=2.0918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.054491339251399e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001929081161506474, 'last_eos_top1': 0} +step=67250 lr=6.000000e-04 loss=2.4180 ce=2.4180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.581610134366201e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017273209523409605, 'last_eos_top1': 0} +step=67300 lr=6.000000e-04 loss=3.0526 ce=3.0526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.27375606604619e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017228471115231514, 'last_eos_top1': 0} +step=67350 lr=6.000000e-04 loss=2.4760 ce=2.4760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9066915885778144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00168694369494915, 'last_eos_top1': 0} +step=67400 lr=6.000000e-04 loss=2.1990 ce=2.1990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.541728907905053e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020374294836074114, 'last_eos_top1': 0} +step=67450 lr=6.000000e-04 loss=2.6686 ce=2.6686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3667163329955656e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022200006060302258, 'last_eos_top1': 0} +step=67500 lr=6.000000e-04 loss=2.8398 ce=2.8398 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.687493052799255e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026876365300267935, 'last_eos_top1': 0} +step=67550 lr=6.000000e-04 loss=2.2001 ce=2.2001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.653673461987637e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037759370170533657, 'last_eos_top1': 0} +step=67600 lr=6.000000e-04 loss=2.8619 ce=2.8619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.595474714122247e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016118785133585334, 'last_eos_top1': 0} +step=67650 lr=6.000000e-04 loss=2.3546 ce=2.3546 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.00003326224396e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014111156342551112, 'last_eos_top1': 0} +step=67700 lr=6.000000e-04 loss=2.4703 ce=2.4703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.312103429692797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021268874406814575, 'last_eos_top1': 0} +step=67750 lr=6.000000e-04 loss=2.2469 ce=2.2469 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2114525816286914e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021523390896618366, 'last_eos_top1': 0} +step=67800 lr=6.000000e-04 loss=2.2591 ce=2.2591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.019496398337651e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002680857665836811, 'last_eos_top1': 0} +step=67850 lr=6.000000e-04 loss=3.3674 ce=3.3674 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9616356439655647e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004131700843572617, 'last_eos_top1': 0} +step=67900 lr=6.000000e-04 loss=2.1716 ce=2.1716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8118122170562856e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004191986285150051, 'last_eos_top1': 0} +step=67950 lr=6.000000e-04 loss=2.8842 ce=2.8842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.29122181533603e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031323032453656197, 'last_eos_top1': 0} +step=68000 lr=6.000000e-04 loss=3.0540 ce=3.0540 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.230435555044096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031936271116137505, 'last_eos_top1': 0} +step=68050 lr=6.000000e-04 loss=2.9689 ce=2.9689 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3344860892393626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002133916597813368, 'last_eos_top1': 0} +step=68100 lr=6.000000e-04 loss=2.5909 ce=2.5909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.999487358261831e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003112069796770811, 'last_eos_top1': 0} +step=68150 lr=6.000000e-04 loss=2.3671 ce=2.3671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.27197618491482e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038184323348104954, 'last_eos_top1': 0} +step=68200 lr=6.000000e-04 loss=2.4427 ce=2.4427 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3866547204961535e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031285095028579235, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=4 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=3] docs=1100000 rows=1091809 buffered=8191 part=openwebtext-t5 + +step=68250 lr=6.000000e-04 loss=2.3664 ce=2.3664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0036921594291925e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019688946194946766, 'last_eos_top1': 0} +step=68300 lr=6.000000e-04 loss=2.2076 ce=2.2076 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1437541616032831e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003039628965780139, 'last_eos_top1': 0} +step=68350 lr=6.000000e-04 loss=2.5526 ce=2.5526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1680743227771018e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027869411278516054, 'last_eos_top1': 0} +step=68400 lr=6.000000e-04 loss=2.7636 ce=2.7636 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0778490832308307e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027612480334937572, 'last_eos_top1': 0} +step=68450 lr=6.000000e-04 loss=2.5226 ce=2.5226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.083246338064782e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002313850913196802, 'last_eos_top1': 0} +step=68500 lr=6.000000e-04 loss=2.0001 ce=2.0001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4302444469649345e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003856020513921976, 'last_eos_top1': 0} +step=68550 lr=6.000000e-04 loss=2.4958 ce=2.4958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.929490048903972e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003738964442163706, 'last_eos_top1': 0} +step=68600 lr=6.000000e-04 loss=2.1679 ce=2.1679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.092367114732042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027369463350623846, 'last_eos_top1': 0} +step=68650 lr=6.000000e-04 loss=2.3791 ce=2.3791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.929175414261408e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003153592348098755, 'last_eos_top1': 0} +step=68700 lr=6.000000e-04 loss=2.6540 ce=2.6540 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0986286472179927e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004124098923057318, 'last_eos_top1': 0} +step=68750 lr=6.000000e-04 loss=2.6039 ce=2.6039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1892476070206612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004353104624897242, 'last_eos_top1': 0} +step=68800 lr=6.000000e-04 loss=2.1299 ce=2.1299 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.036877261474729e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035946595016866922, 'last_eos_top1': 0} +step=68850 lr=6.000000e-04 loss=3.2920 ce=3.2920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.164154107565992e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034880873281508684, 'last_eos_top1': 0} +step=68900 lr=6.000000e-04 loss=2.2588 ce=2.2588 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1759210792661179e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025612395256757736, 'last_eos_top1': 0} +step=68950 lr=6.000000e-04 loss=1.7836 ce=1.7836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.554076324187918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017093936912715435, 'last_eos_top1': 0} +step=69000 lr=6.000000e-04 loss=2.3996 ce=2.3996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3473133549268823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017601887229830027, 'last_eos_top1': 0} +step=69050 lr=6.000000e-04 loss=2.2730 ce=2.2730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5946621462935582e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027293104212731123, 'last_eos_top1': 0} +step=69100 lr=6.000000e-04 loss=2.5595 ce=2.5595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.178729093633592e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037403078749775887, 'last_eos_top1': 0} +step=69150 lr=6.000000e-04 loss=2.6374 ce=2.6374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.574980994220823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003425080794841051, 'last_eos_top1': 0} +step=69200 lr=6.000000e-04 loss=2.6237 ce=2.6237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.736710692057386e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015860656276345253, 'last_eos_top1': 0} +step=69250 lr=6.000000e-04 loss=2.5352 ce=2.5352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4615176951338071e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019975716713815928, 'last_eos_top1': 0} +step=69300 lr=6.000000e-04 loss=2.8526 ce=2.8526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.111696737003513e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031878717709332705, 'last_eos_top1': 0} +step=69350 lr=6.000000e-04 loss=2.7020 ce=2.7020 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.081785719899926e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005645601777359843, 'last_eos_top1': 0} +step=69400 lr=6.000000e-04 loss=2.0758 ce=2.0758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5197549146250822e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002556470688432455, 'last_eos_top1': 0} +step=69450 lr=6.000000e-04 loss=2.1978 ce=2.1978 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.343834628642071e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024106570053845644, 'last_eos_top1': 0} +step=69500 lr=6.000000e-04 loss=2.4338 ce=2.4338 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.276564014726318e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028518543113023043, 'last_eos_top1': 0} +step=69550 lr=6.000000e-04 loss=2.6704 ce=2.6704 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.34368711366551e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018468458438292146, 'last_eos_top1': 0} +step=69600 lr=6.000000e-04 loss=2.2681 ce=2.2681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0280233002267778e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015872179064899683, 'last_eos_top1': 0} +step=69650 lr=6.000000e-04 loss=2.3756 ce=2.3756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.243824565084651e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013342739548534155, 'last_eos_top1': 0} +step=69700 lr=6.000000e-04 loss=2.8867 ce=2.8867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1020245185354725e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012954018311575055, 'last_eos_top1': 0} +step=69750 lr=6.000000e-04 loss=1.7700 ce=1.7700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3251413292891812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009034627582877874, 'last_eos_top1': 0} +step=69800 lr=6.000000e-04 loss=2.9657 ce=2.9657 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.203873347956687e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013807826908305287, 'last_eos_top1': 0} +step=69850 lr=6.000000e-04 loss=2.5764 ce=2.5764 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2590375010622665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003187880211044103, 'last_eos_top1': 0} +step=69900 lr=6.000000e-04 loss=2.5032 ce=2.5032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.763978257775307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.935722194612026e-05, 'last_eos_top1': 0} +step=69950 lr=6.000000e-04 loss=2.6922 ce=2.6922 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.717929863953032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031150237191468477, 'last_eos_top1': 0} +step=70000 lr=6.000000e-04 loss=2.3536 ce=2.3536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2678132407017983e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006326734437607229, 'last_eos_top1': 0} +step=70050 lr=6.000000e-04 loss=2.7146 ce=2.7146 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2087192772014532e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018911913502961397, 'last_eos_top1': 0} +step=70100 lr=6.000000e-04 loss=2.3808 ce=2.3808 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.075831258087419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014350633136928082, 'last_eos_top1': 0} +step=70150 lr=6.000000e-04 loss=2.5357 ce=2.5357 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.350563803105615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00223838584497571, 'last_eos_top1': 0} +step=70200 lr=6.000000e-04 loss=2.4335 ce=2.4335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0579042029567063e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005883972626179457, 'last_eos_top1': 0} +step=70250 lr=6.000000e-04 loss=2.1870 ce=2.1870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.37050446937792e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001956577878445387, 'last_eos_top1': 0} +step=70300 lr=6.000000e-04 loss=2.7096 ce=2.7096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1301547096518334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002339799189940095, 'last_eos_top1': 0} +step=70350 lr=6.000000e-04 loss=2.6789 ce=2.6789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.856340067111887e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003542326739989221, 'last_eos_top1': 0} +step=70400 lr=6.000000e-04 loss=3.0649 ce=3.0649 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.77429317572387e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025596721097826958, 'last_eos_top1': 0} +step=70450 lr=6.000000e-04 loss=2.0604 ce=2.0604 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.711809641681612e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015501121524721384, 'last_eos_top1': 0} +step=70500 lr=6.000000e-04 loss=2.8463 ce=2.8463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2938797226524912e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022639459930360317, 'last_eos_top1': 0} +step=70550 lr=6.000000e-04 loss=2.3589 ce=2.3589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.968592631688807e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010069308336824179, 'last_eos_top1': 0} +step=70600 lr=6.000000e-04 loss=2.6364 ce=2.6364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0789785847009625e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022058826871216297, 'last_eos_top1': 0} +step=70650 lr=6.000000e-04 loss=3.3597 ce=3.3597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.241215593647212e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004940629005432129, 'last_eos_top1': 0} +step=70700 lr=6.000000e-04 loss=2.9726 ce=2.9726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.029040949011687e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005311862332746387, 'last_eos_top1': 0} +step=70750 lr=6.000000e-04 loss=2.1813 ce=2.1813 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.119747129356256e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026874798350036144, 'last_eos_top1': 0} +step=70800 lr=6.000000e-04 loss=2.6101 ce=2.6101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.685406904376578e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033427407033741474, 'last_eos_top1': 0} +step=70850 lr=6.000000e-04 loss=2.4826 ce=2.4826 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.6251026257523336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004498663474805653, 'last_eos_top1': 0} +step=70900 lr=6.000000e-04 loss=2.0623 ce=2.0623 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.827104127500206e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019788439385592937, 'last_eos_top1': 0} +step=70950 lr=6.000000e-04 loss=2.5468 ce=2.5468 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.241522896743845e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029190853820182383, 'last_eos_top1': 0} +step=71000 lr=6.000000e-04 loss=2.2560 ce=2.2560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.695051080081612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003376216161996126, 'last_eos_top1': 0} +step=71050 lr=6.000000e-04 loss=2.7992 ce=2.7992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9841008906951174e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001833256334066391, 'last_eos_top1': 0} +step=71100 lr=6.000000e-04 loss=2.1532 ce=2.1532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2164164218120277e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002386617474257946, 'last_eos_top1': 0} +step=71150 lr=6.000000e-04 loss=2.7433 ce=2.7433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0571795175783336e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025401986204087734, 'last_eos_top1': 0} +step=71200 lr=6.000000e-04 loss=2.1446 ce=2.1446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1474359275307506e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003302615601569414, 'last_eos_top1': 0} +step=71250 lr=6.000000e-04 loss=2.4732 ce=2.4732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.318081432254985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003226665547117591, 'last_eos_top1': 0} +step=71300 lr=6.000000e-04 loss=2.3905 ce=2.3905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.367227469017962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003407704411074519, 'last_eos_top1': 0} +step=71350 lr=6.000000e-04 loss=2.7787 ce=2.7787 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.632485873415135e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012849970953539014, 'last_eos_top1': 0} +step=71400 lr=6.000000e-04 loss=2.1115 ce=2.1115 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.258119462174363e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026948859449476004, 'last_eos_top1': 0} +step=71450 lr=6.000000e-04 loss=2.6493 ce=2.6493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.987713954411447e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019902531057596207, 'last_eos_top1': 0} +step=71500 lr=6.000000e-04 loss=2.6355 ce=2.6355 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0214603611966595e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00174736138433218, 'last_eos_top1': 0} +step=71550 lr=6.000000e-04 loss=2.4870 ce=2.4870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.833944406826049e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004008936230093241, 'last_eos_top1': 0} +step=71600 lr=6.000000e-04 loss=2.1352 ce=2.1352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0546167686698027e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002175228437408805, 'last_eos_top1': 0} +step=71650 lr=6.000000e-04 loss=2.9086 ce=2.9086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.394316187652294e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002300737891346216, 'last_eos_top1': 0} +step=71700 lr=6.000000e-04 loss=3.1872 ce=3.1872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9261864508735016e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019514563027769327, 'last_eos_top1': 0} +step=71750 lr=6.000000e-04 loss=2.4384 ce=2.4384 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.178273669211194e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001815066090784967, 'last_eos_top1': 0} +step=71800 lr=6.000000e-04 loss=2.6975 ce=2.6975 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5802550478838384e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011114779626950622, 'last_eos_top1': 0} +step=71850 lr=6.000000e-04 loss=2.6997 ce=2.6997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3885402697487734e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016462183557450771, 'last_eos_top1': 0} +step=71900 lr=6.000000e-04 loss=2.7200 ce=2.7200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.186851467442466e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00156124634668231, 'last_eos_top1': 0} +step=71950 lr=6.000000e-04 loss=2.6789 ce=2.6789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.175992929347558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002078176476061344, 'last_eos_top1': 0} +step=72000 lr=6.000000e-04 loss=2.6962 ce=2.6962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.034685379243456e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011126019526273012, 'last_eos_top1': 0} +step=72050 lr=6.000000e-04 loss=2.5789 ce=2.5789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1833253665827215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010156355565413833, 'last_eos_top1': 0} +step=72100 lr=6.000000e-04 loss=2.7878 ce=2.7878 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0346398084948305e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001379758701659739, 'last_eos_top1': 0} +step=72150 lr=6.000000e-04 loss=2.2664 ce=2.2664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1581649232539348e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002323681954294443, 'last_eos_top1': 0} +step=72200 lr=6.000000e-04 loss=2.3909 ce=2.3909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.700093869469129e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002186473924666643, 'last_eos_top1': 0} +step=72250 lr=6.000000e-04 loss=1.8477 ce=1.8477 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.080175115930615e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024036383256316185, 'last_eos_top1': 0} +step=72300 lr=6.000000e-04 loss=2.8708 ce=2.8708 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.759991381637519e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009042741148732603, 'last_eos_top1': 0} +step=72350 lr=6.000000e-04 loss=2.4530 ce=2.4530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2100059140939265e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002312570810317993, 'last_eos_top1': 0} +step=72400 lr=6.000000e-04 loss=2.3913 ce=2.3913 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.801615144358948e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001895685913041234, 'last_eos_top1': 0} +step=72450 lr=6.000000e-04 loss=3.0133 ce=3.0133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.52246375870891e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023544125724583864, 'last_eos_top1': 0} +step=72500 lr=6.000000e-04 loss=2.9113 ce=2.9113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.493006599659566e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003795990254729986, 'last_eos_top1': 0} +step=72550 lr=6.000000e-04 loss=2.2767 ce=2.2767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.788730494619813e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029275664128363132, 'last_eos_top1': 0} +step=72600 lr=6.000000e-04 loss=2.8088 ce=2.8088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.509248800692149e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004292267374694347, 'last_eos_top1': 0} +step=72650 lr=6.000000e-04 loss=2.7911 ce=2.7911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0093407581734937e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004866050090640783, 'last_eos_top1': 0} +step=72700 lr=6.000000e-04 loss=2.8416 ce=2.8416 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1688101949403062e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039759185165166855, 'last_eos_top1': 0} +step=72750 lr=6.000000e-04 loss=2.5698 ce=2.5698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1526073649292812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003240724327042699, 'last_eos_top1': 0} +step=72800 lr=6.000000e-04 loss=2.6198 ce=2.6198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.191348463995382e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010811997344717383, 'last_eos_top1': 0} +step=72850 lr=6.000000e-04 loss=2.8281 ce=2.8281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1364858437445946e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016501480713486671, 'last_eos_top1': 0} +step=72900 lr=6.000000e-04 loss=2.6482 ce=2.6482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4173390809446573e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001462629297748208, 'last_eos_top1': 0} +step=72950 lr=6.000000e-04 loss=2.6798 ce=2.6798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.7529682534514e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023523056879639626, 'last_eos_top1': 0} +step=73000 lr=6.000000e-04 loss=2.1922 ce=2.1922 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.373270247830078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019967667758464813, 'last_eos_top1': 0} +step=73050 lr=6.000000e-04 loss=2.7949 ce=2.7949 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2690857804263942e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016885900404304266, 'last_eos_top1': 0} +step=73100 lr=6.000000e-04 loss=2.9679 ce=2.9679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9031120801810175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025714891962707043, 'last_eos_top1': 0} +step=73150 lr=6.000000e-04 loss=2.2415 ce=2.2415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.32101113194949e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017681909957900643, 'last_eos_top1': 0} +step=73200 lr=6.000000e-04 loss=2.6428 ce=2.6428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2870495993411168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011622690362855792, 'last_eos_top1': 0} +step=73250 lr=6.000000e-04 loss=3.6016 ce=3.6016 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.531186899228487e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011189451906830072, 'last_eos_top1': 0} +step=73300 lr=6.000000e-04 loss=2.0703 ce=2.0703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2320118912612088e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020407983101904392, 'last_eos_top1': 0} +step=73350 lr=6.000000e-04 loss=2.3806 ce=2.3806 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.231674994225614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002373204566538334, 'last_eos_top1': 0} +step=73400 lr=6.000000e-04 loss=2.2073 ce=2.2073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.566920663113706e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018234697636216879, 'last_eos_top1': 0} +step=73450 lr=6.000000e-04 loss=2.6875 ce=2.6875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.377261004468892e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026476329658180475, 'last_eos_top1': 0} +step=73500 lr=6.000000e-04 loss=2.1864 ce=2.1864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.223779670719523e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001215117983520031, 'last_eos_top1': 0} +step=73550 lr=6.000000e-04 loss=2.7572 ce=2.7572 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3993521861266345e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007604686543345451, 'last_eos_top1': 0} +step=73600 lr=6.000000e-04 loss=2.8147 ce=2.8147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1922118321526796e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000587546790484339, 'last_eos_top1': 0} +step=73650 lr=6.000000e-04 loss=2.3214 ce=2.3214 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.887517884024419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007211362244561315, 'last_eos_top1': 0} +step=73700 lr=6.000000e-04 loss=2.8812 ce=2.8812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.278906196006574e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005768149858340621, 'last_eos_top1': 0} +step=73750 lr=6.000000e-04 loss=2.7475 ce=2.7475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5299698134185746e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000949613400734961, 'last_eos_top1': 0} +step=73800 lr=6.000000e-04 loss=2.1835 ce=2.1835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.068726760422578e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006651841686107218, 'last_eos_top1': 0} +step=73850 lr=6.000000e-04 loss=1.9017 ce=1.9017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8062630008207634e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001513784285634756, 'last_eos_top1': 0} +step=73900 lr=6.000000e-04 loss=2.8705 ce=2.8705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.091439389711013e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007242477731779218, 'last_eos_top1': 0} +step=73950 lr=6.000000e-04 loss=2.9099 ce=2.9099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6602631148998626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001148413633927703, 'last_eos_top1': 0} +step=74000 lr=6.000000e-04 loss=2.7285 ce=2.7285 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8568076484370977e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008003519615158439, 'last_eos_top1': 0} +step=74050 lr=6.000000e-04 loss=2.2369 ce=2.2369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2385037734929938e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011392318410798907, 'last_eos_top1': 0} +step=74100 lr=6.000000e-04 loss=2.4051 ce=2.4051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3535385733121075e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010567617136985064, 'last_eos_top1': 0} +step=74150 lr=6.000000e-04 loss=2.8352 ce=2.8352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.655338201089762e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008116801036521792, 'last_eos_top1': 0} +step=74200 lr=6.000000e-04 loss=2.8226 ce=2.8226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2607993994606659e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006047610659152269, 'last_eos_top1': 0} +step=74250 lr=6.000000e-04 loss=2.8596 ce=2.8596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.785485649947077e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028300125268287957, 'last_eos_top1': 0} +step=74300 lr=6.000000e-04 loss=2.3682 ce=2.3682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6362275346182287e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002823316026479006, 'last_eos_top1': 0} +step=74350 lr=6.000000e-04 loss=2.4611 ce=2.4611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.512064020265825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009529811795800924, 'last_eos_top1': 0} +step=74400 lr=6.000000e-04 loss=3.0381 ce=3.0381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1974079825449735e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000645441235974431, 'last_eos_top1': 0} +step=74450 lr=6.000000e-04 loss=2.7408 ce=2.7408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4535020454786718e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005225432105362415, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1200000 rows=1191809 buffered=8191 part=openwebtext-t5 +step=74500 lr=6.000000e-04 loss=2.2554 ce=2.2554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2191419045848306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047323855687864125, 'last_eos_top1': 0} +step=74550 lr=6.000000e-04 loss=2.9197 ce=2.9197 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.236115895153489e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000594807555899024, 'last_eos_top1': 0} +step=74600 lr=6.000000e-04 loss=3.0671 ce=3.0671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.889431460876949e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007528619607910514, 'last_eos_top1': 0} +step=74650 lr=6.000000e-04 loss=2.3985 ce=2.3985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0622072598780505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005751632270403206, 'last_eos_top1': 0} +step=74700 lr=6.000000e-04 loss=2.4645 ce=2.4645 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.775198122952133e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006541780894622207, 'last_eos_top1': 0} +step=74750 lr=6.000000e-04 loss=2.6596 ce=2.6596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1432479115901515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000794913386926055, 'last_eos_top1': 0} +step=74800 lr=6.000000e-04 loss=2.1689 ce=2.1689 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1688291124301031e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009790877811610699, 'last_eos_top1': 0} +step=74850 lr=6.000000e-04 loss=2.3136 ce=2.3136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.796881047601346e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007160011446103454, 'last_eos_top1': 0} +step=74900 lr=6.000000e-04 loss=3.0352 ce=3.0352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.37875995784998e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045995801337994635, 'last_eos_top1': 0} +step=74950 lr=6.000000e-04 loss=2.6249 ce=2.6249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6982741726678796e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015304719563573599, 'last_eos_top1': 0} +step=75000 lr=6.000000e-04 loss=2.6752 ce=2.6752 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.255617674672976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013081233482807875, 'last_eos_top1': 0} +step=75050 lr=6.000000e-04 loss=2.5633 ce=2.5633 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.116267594625242e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00048496495583094656, 'last_eos_top1': 0} +step=75100 lr=6.000000e-04 loss=2.4698 ce=2.4698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2391270502121188e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000367571075912565, 'last_eos_top1': 0} +step=75150 lr=6.000000e-04 loss=2.2562 ce=2.2562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.090608728001826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002857849467545748, 'last_eos_top1': 0} +step=75200 lr=6.000000e-04 loss=2.5662 ce=2.5662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.903793397825211e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003796373202931136, 'last_eos_top1': 0} +step=75250 lr=6.000000e-04 loss=2.7666 ce=2.7666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.702011084300466e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005736270686611533, 'last_eos_top1': 0} +step=75300 lr=6.000000e-04 loss=2.6066 ce=2.6066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.6185140137095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007025368977338076, 'last_eos_top1': 0} +step=75350 lr=6.000000e-04 loss=2.9714 ce=2.9714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.976952242141124e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005322443321347237, 'last_eos_top1': 0} +step=75400 lr=6.000000e-04 loss=2.9019 ce=2.9019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.062035936338361e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008253023261204362, 'last_eos_top1': 0} +step=75450 lr=6.000000e-04 loss=2.6319 ce=2.6319 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2248516213730909e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009084336925297976, 'last_eos_top1': 0} +step=75500 lr=6.000000e-04 loss=2.6765 ce=2.6765 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2846307880827226e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008587889606133103, 'last_eos_top1': 0} +step=75550 lr=6.000000e-04 loss=2.6880 ce=2.6880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.192840373842046e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009559840545989573, 'last_eos_top1': 0} +step=75600 lr=6.000000e-04 loss=2.2291 ce=2.2291 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.307067112880759e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00063499272800982, 'last_eos_top1': 0} +step=75650 lr=6.000000e-04 loss=2.4300 ce=2.4300 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.897201218525879e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000961461802944541, 'last_eos_top1': 0} +step=75700 lr=6.000000e-04 loss=2.6369 ce=2.6369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.474311102356296e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012384140864014626, 'last_eos_top1': 0} +step=75750 lr=6.000000e-04 loss=2.3536 ce=2.3536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4961266060709022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013389254454523325, 'last_eos_top1': 0} +step=75800 lr=6.000000e-04 loss=2.3447 ce=2.3447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0397171940421686e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014555923407897353, 'last_eos_top1': 0} +step=75850 lr=6.000000e-04 loss=2.1168 ce=2.1168 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7997368306387216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012165113585069776, 'last_eos_top1': 0} +step=75900 lr=6.000000e-04 loss=3.1473 ce=3.1473 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0598839910235256e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007788974326103926, 'last_eos_top1': 0} +step=75950 lr=6.000000e-04 loss=2.4831 ce=2.4831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3076542018097825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006920087616890669, 'last_eos_top1': 0} +step=76000 lr=6.000000e-04 loss=2.7129 ce=2.7129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.264937782252673e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013545019319280982, 'last_eos_top1': 0} +step=76050 lr=6.000000e-04 loss=2.5814 ce=2.5814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4761855456745252e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001551965018734336, 'last_eos_top1': 0} +step=76100 lr=6.000000e-04 loss=2.7946 ce=2.7946 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7084646970033646e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011426445562392473, 'last_eos_top1': 0} +step=76150 lr=6.000000e-04 loss=2.4384 ce=2.4384 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.322924072155729e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008214856497943401, 'last_eos_top1': 0} +step=76200 lr=6.000000e-04 loss=2.9795 ce=2.9795 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0041923815151677e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009857115801423788, 'last_eos_top1': 0} +step=76250 lr=6.000000e-04 loss=2.8725 ce=2.8725 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7264053894905373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017485527787357569, 'last_eos_top1': 0} +step=76300 lr=6.000000e-04 loss=2.5069 ce=2.5069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5528956282651052e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000806402531452477, 'last_eos_top1': 0} +step=76350 lr=6.000000e-04 loss=2.9095 ce=2.9095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.218838047469035e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010306828189641237, 'last_eos_top1': 0} +step=76400 lr=6.000000e-04 loss=1.9889 ce=1.9889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.966138476447668e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012909967917948961, 'last_eos_top1': 0} +step=76450 lr=6.000000e-04 loss=2.2128 ce=2.2128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.575396011816338e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007499837083742023, 'last_eos_top1': 0} +step=76500 lr=6.000000e-04 loss=2.4883 ce=2.4883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.573871243948815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005800781073048711, 'last_eos_top1': 0} +step=76550 lr=6.000000e-04 loss=2.9145 ce=2.9145 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.199971666210331e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047311175148934126, 'last_eos_top1': 0} +step=76600 lr=6.000000e-04 loss=3.0489 ce=3.0489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.904926830320619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000639983220025897, 'last_eos_top1': 0} +step=76650 lr=6.000000e-04 loss=2.1546 ce=2.1546 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5596699086017907e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000740524847060442, 'last_eos_top1': 0} +step=76700 lr=6.000000e-04 loss=2.5471 ce=2.5471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9542694644769654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014337407192215323, 'last_eos_top1': 0} +step=76750 lr=6.000000e-04 loss=2.5323 ce=2.5323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2460912077804096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024248724803328514, 'last_eos_top1': 0} +step=76800 lr=6.000000e-04 loss=2.7415 ce=2.7415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.615374640095979e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019487475510686636, 'last_eos_top1': 0} +step=76850 lr=6.000000e-04 loss=2.8531 ce=2.8531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3304073036124464e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002082844264805317, 'last_eos_top1': 0} +step=76900 lr=6.000000e-04 loss=2.2692 ce=2.2692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.628089325502515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019182011019438505, 'last_eos_top1': 0} +step=76950 lr=6.000000e-04 loss=2.6268 ce=2.6268 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6949028577073477e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001024533761665225, 'last_eos_top1': 0} +step=77000 lr=6.000000e-04 loss=2.1276 ce=2.1276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0678656154777855e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014894346240907907, 'last_eos_top1': 0} +step=77050 lr=6.000000e-04 loss=2.5023 ce=2.5023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9878769307979383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001644409028813243, 'last_eos_top1': 0} +step=77100 lr=6.000000e-04 loss=2.5286 ce=2.5286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5980978787411004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006813437794335186, 'last_eos_top1': 0} +step=77150 lr=6.000000e-04 loss=2.3771 ce=2.3771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3714216038351879e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005164986941963434, 'last_eos_top1': 0} +step=77200 lr=6.000000e-04 loss=2.8309 ce=2.8309 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5741508579812944e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003178720362484455, 'last_eos_top1': 0} +step=77250 lr=6.000000e-04 loss=2.3291 ce=2.3291 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.412930189166218e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022816512500867248, 'last_eos_top1': 0} +step=77300 lr=6.000000e-04 loss=2.4273 ce=2.4273 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5104938938748091e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004339060396887362, 'last_eos_top1': 0} +step=77350 lr=6.000000e-04 loss=2.9152 ce=2.9152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1495230864966288e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015102711040526628, 'last_eos_top1': 0} +step=77400 lr=6.000000e-04 loss=2.8415 ce=2.8415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.04222396860132e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006362541462294757, 'last_eos_top1': 0} +step=77450 lr=6.000000e-04 loss=2.7086 ce=2.7086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.749420314619783e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007263387669809163, 'last_eos_top1': 0} +step=77500 lr=6.000000e-04 loss=2.7595 ce=2.7595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5110934327822179e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023147220781538635, 'last_eos_top1': 0} +step=77550 lr=6.000000e-04 loss=2.9986 ce=2.9986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6838950614328496e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004940620856359601, 'last_eos_top1': 0} +step=77600 lr=6.000000e-04 loss=2.8464 ce=2.8464 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0515394023968838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001652813283726573, 'last_eos_top1': 0} +step=77650 lr=6.000000e-04 loss=2.8762 ce=2.8762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6348587450920604e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014320819173008204, 'last_eos_top1': 0} +step=77700 lr=6.000000e-04 loss=2.7358 ce=2.7358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.141104985435959e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000829532218631357, 'last_eos_top1': 0} +step=77750 lr=6.000000e-04 loss=2.2361 ce=2.2361 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1930864275200292e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005975716048851609, 'last_eos_top1': 0} +step=77800 lr=6.000000e-04 loss=2.0758 ce=2.0758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.189000593323726e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006621512584388256, 'last_eos_top1': 0} +step=77850 lr=6.000000e-04 loss=2.9095 ce=2.9095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.616282679606229e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001336436253041029, 'last_eos_top1': 0} +step=77900 lr=6.000000e-04 loss=2.1827 ce=2.1827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.750401214754675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012436241377145052, 'last_eos_top1': 0} +step=77950 lr=6.000000e-04 loss=2.9682 ce=2.9682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.001084329909645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010549324797466397, 'last_eos_top1': 0} +step=78000 lr=6.000000e-04 loss=2.8383 ce=2.8383 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.557372888782993e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015061895828694105, 'last_eos_top1': 0} +step=78050 lr=6.000000e-04 loss=2.6691 ce=2.6691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.003355105756782e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008884286507964134, 'last_eos_top1': 0} +step=78100 lr=6.000000e-04 loss=3.3180 ce=3.3180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3624041205039248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010785835329443216, 'last_eos_top1': 0} +step=78150 lr=6.000000e-04 loss=3.0936 ce=3.0936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.525643842294812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018931294325739145, 'last_eos_top1': 0} +step=78200 lr=6.000000e-04 loss=2.6467 ce=2.6467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5788791642989963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013909165281802416, 'last_eos_top1': 0} +step=78250 lr=6.000000e-04 loss=3.0148 ce=3.0148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6686617527739145e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014958870597183704, 'last_eos_top1': 0} +step=78300 lr=6.000000e-04 loss=2.5023 ce=2.5023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.588790473761037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001599614741280675, 'last_eos_top1': 0} +step=78350 lr=6.000000e-04 loss=2.7300 ce=2.7300 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3354547010967508e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002100238110870123, 'last_eos_top1': 0} +step=78400 lr=6.000000e-04 loss=2.6547 ce=2.6547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.283856131020002e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001946698292158544, 'last_eos_top1': 0} +step=78450 lr=6.000000e-04 loss=2.7125 ce=2.7125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.644391886657104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016583639662712812, 'last_eos_top1': 0} +step=78500 lr=6.000000e-04 loss=2.3592 ce=2.3592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0804247722262517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010276851244270802, 'last_eos_top1': 0} +step=78550 lr=6.000000e-04 loss=2.7224 ce=2.7224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.723504263092764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012326983269304037, 'last_eos_top1': 0} +step=78600 lr=6.000000e-04 loss=2.1085 ce=2.1085 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5059522411320359e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015245857648551464, 'last_eos_top1': 0} +step=78650 lr=6.000000e-04 loss=3.2279 ce=3.2279 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2641446776106022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012473822571337223, 'last_eos_top1': 0} +step=78700 lr=6.000000e-04 loss=2.5714 ce=2.5714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.077083601965569e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001345910830423236, 'last_eos_top1': 0} +step=78750 lr=6.000000e-04 loss=2.1049 ce=2.1049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.942567541846074e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001748665003105998, 'last_eos_top1': 0} +step=78800 lr=6.000000e-04 loss=2.4189 ce=2.4189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0602123438729905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001723982160910964, 'last_eos_top1': 0} +step=78850 lr=6.000000e-04 loss=2.6619 ce=2.6619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2213060472276993e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002041612286120653, 'last_eos_top1': 0} +step=78900 lr=6.000000e-04 loss=2.9566 ce=2.9566 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7517632992821746e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002009991090744734, 'last_eos_top1': 0} +step=78950 lr=6.000000e-04 loss=2.7893 ce=2.7893 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.921069482108578e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018572031985968351, 'last_eos_top1': 0} +step=79000 lr=6.000000e-04 loss=2.4982 ce=2.4982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.571487003588118e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021643717773258686, 'last_eos_top1': 0} +step=79050 lr=6.000000e-04 loss=2.4786 ce=2.4786 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0167839466012083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001644281786866486, 'last_eos_top1': 0} +step=79100 lr=6.000000e-04 loss=2.6885 ce=2.6885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2299172340135556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005707367090508342, 'last_eos_top1': 0} +step=79150 lr=6.000000e-04 loss=2.9185 ce=2.9185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5540432286798023e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018786460859701037, 'last_eos_top1': 0} +step=79200 lr=6.000000e-04 loss=2.6179 ce=2.6179 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6441470506833866e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001528859487734735, 'last_eos_top1': 0} +step=79250 lr=6.000000e-04 loss=2.3281 ce=2.3281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0525221771094948e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012874422827735543, 'last_eos_top1': 0} +step=79300 lr=6.000000e-04 loss=2.5776 ce=2.5776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.311833668500185e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001140236621722579, 'last_eos_top1': 0} +step=79350 lr=6.000000e-04 loss=2.2227 ce=2.2227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.626895391382277e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001539092743769288, 'last_eos_top1': 0} +step=79400 lr=6.000000e-04 loss=2.6227 ce=2.6227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.466083267587237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015176457818597555, 'last_eos_top1': 0} +step=79450 lr=6.000000e-04 loss=2.4410 ce=2.4410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8014410670730285e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019662503618746996, 'last_eos_top1': 0} +step=79500 lr=6.000000e-04 loss=2.5075 ce=2.5075 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.766766217770055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003113158978521824, 'last_eos_top1': 0} +step=79550 lr=6.000000e-04 loss=2.3562 ce=2.3562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5609308320563287e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020870049484074116, 'last_eos_top1': 0} +step=79600 lr=6.000000e-04 loss=2.7254 ce=2.7254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7825208487920463e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026604854501783848, 'last_eos_top1': 0} +step=79650 lr=6.000000e-04 loss=2.6630 ce=2.6630 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3638097698276397e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002417241223156452, 'last_eos_top1': 0} +step=79700 lr=6.000000e-04 loss=2.7859 ce=2.7859 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.245375799247995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021088789217174053, 'last_eos_top1': 0} +step=79750 lr=6.000000e-04 loss=2.6050 ce=2.6050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.01352359505836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002068163827061653, 'last_eos_top1': 0} +step=79800 lr=6.000000e-04 loss=3.0517 ce=3.0517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.290423097088933e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013901866041123867, 'last_eos_top1': 0} +step=79850 lr=6.000000e-04 loss=1.9170 ce=1.9170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6421359759988263e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002233427483588457, 'last_eos_top1': 0} +step=79900 lr=6.000000e-04 loss=2.3736 ce=2.3736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.375280231120996e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019412845140323043, 'last_eos_top1': 0} +step=79950 lr=6.000000e-04 loss=2.5334 ce=2.5334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.220892954734154e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023069416638463736, 'last_eos_top1': 0} +step=80000 lr=6.000000e-04 loss=2.3529 ce=2.3529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3489516277331859e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016094697639346123, 'last_eos_top1': 0} +step=80050 lr=6.000000e-04 loss=3.2354 ce=3.2354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5788229575264268e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015103621408343315, 'last_eos_top1': 0} +step=80100 lr=6.000000e-04 loss=2.2187 ce=2.2187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.354219400847796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013209439348429441, 'last_eos_top1': 0} +step=80150 lr=6.000000e-04 loss=2.4334 ce=2.4334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2173257346148603e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002002013847231865, 'last_eos_top1': 0} +step=80200 lr=6.000000e-04 loss=1.9875 ce=1.9875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7651849702815525e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014219016302376986, 'last_eos_top1': 0} +step=80250 lr=6.000000e-04 loss=2.7677 ce=2.7677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1099704352091067e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002539545064792037, 'last_eos_top1': 0} +step=80300 lr=6.000000e-04 loss=2.8196 ce=2.8196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4199562428984791e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015034065581858158, 'last_eos_top1': 0} +step=80350 lr=6.000000e-04 loss=2.2789 ce=2.2789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7474128981120884e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018384621944278479, 'last_eos_top1': 0} +step=80400 lr=6.000000e-04 loss=2.1150 ce=2.1150 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2474372852011584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009857919067144394, 'last_eos_top1': 0} +step=80450 lr=6.000000e-04 loss=2.7896 ce=2.7896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.23389416129794e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010925454553216696, 'last_eos_top1': 0} +step=80500 lr=6.000000e-04 loss=1.7801 ce=1.7801 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.922425432596356e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010752659291028976, 'last_eos_top1': 0} +step=80550 lr=6.000000e-04 loss=2.3058 ce=2.3058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.905053745256737e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046644348185509443, 'last_eos_top1': 0} +step=80600 lr=6.000000e-04 loss=2.6169 ce=2.6169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2100326532381587e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001798013225197792, 'last_eos_top1': 0} +step=80650 lr=6.000000e-04 loss=2.8971 ce=2.8971 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0030052989313845e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010679815895855427, 'last_eos_top1': 0} +step=80700 lr=6.000000e-04 loss=2.0606 ce=2.0606 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.836229204054689e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029977508820593357, 'last_eos_top1': 0} +[elf_hfds direct rank=6 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1300000 rows=1291809 buffered=8191 part=openwebtext-t5 +step=80750 lr=6.000000e-04 loss=2.4218 ce=2.4218 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.510746632353403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035758765880018473, 'last_eos_top1': 0} +step=80800 lr=6.000000e-04 loss=2.3646 ce=2.3646 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5603813962079585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004675045143812895, 'last_eos_top1': 0} +step=80850 lr=6.000000e-04 loss=2.5096 ce=2.5096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.102817784878425e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005176706239581108, 'last_eos_top1': 0} +step=80900 lr=6.000000e-04 loss=1.8605 ce=1.8605 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.248085810511839e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004363734740763903, 'last_eos_top1': 0} +step=80950 lr=6.000000e-04 loss=1.9488 ce=1.9488 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.815819582086988e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004831927828490734, 'last_eos_top1': 0} +step=81000 lr=6.000000e-04 loss=2.2531 ce=2.2531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.525251253857277e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020515606738626957, 'last_eos_top1': 0} +step=81050 lr=6.000000e-04 loss=2.4671 ce=2.4671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3151125787990168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002303238958120346, 'last_eos_top1': 0} +step=81100 lr=6.000000e-04 loss=2.5545 ce=2.5545 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5297813661163673e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019367567729204893, 'last_eos_top1': 0} +step=81150 lr=6.000000e-04 loss=2.4328 ce=2.4328 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.283225153922103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005001171957701445, 'last_eos_top1': 0} +step=81200 lr=6.000000e-04 loss=2.4090 ce=2.4090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.744875953590963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028718297835439444, 'last_eos_top1': 0} +step=81250 lr=6.000000e-04 loss=2.5629 ce=2.5629 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5163082935032435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00424738135188818, 'last_eos_top1': 0} +step=81300 lr=6.000000e-04 loss=2.5168 ce=2.5168 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.410295590176247e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002966268453747034, 'last_eos_top1': 0} +step=81350 lr=6.000000e-04 loss=2.7035 ce=2.7035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.679868526291102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001227399567142129, 'last_eos_top1': 0} +step=81400 lr=6.000000e-04 loss=2.5276 ce=2.5276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1428739273687825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017899894155561924, 'last_eos_top1': 0} +step=81450 lr=6.000000e-04 loss=2.5667 ce=2.5667 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0293286322848871e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022211000323295593, 'last_eos_top1': 0} +step=81500 lr=6.000000e-04 loss=3.0518 ce=3.0518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0229795407212805e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028358707204461098, 'last_eos_top1': 0} +step=81550 lr=6.000000e-04 loss=2.4331 ce=2.4331 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.809349123504944e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003331893589347601, 'last_eos_top1': 0} +step=81600 lr=6.000000e-04 loss=3.2381 ce=3.2381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.038304890244035e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021775499917566776, 'last_eos_top1': 0} +step=81650 lr=6.000000e-04 loss=2.1730 ce=2.1730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0047031537396833e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016772211529314518, 'last_eos_top1': 0} +step=81700 lr=6.000000e-04 loss=2.5318 ce=2.5318 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.859467586968094e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023686480708420277, 'last_eos_top1': 0} +step=81750 lr=6.000000e-04 loss=2.5517 ce=2.5517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.743064659531228e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025576134212315083, 'last_eos_top1': 0} +step=81800 lr=6.000000e-04 loss=2.8858 ce=2.8858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5521278075757436e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00277443858794868, 'last_eos_top1': 0} +step=81850 lr=6.000000e-04 loss=2.6015 ce=2.6015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8389036742737517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019032900454476476, 'last_eos_top1': 0} +step=81900 lr=6.000000e-04 loss=2.7460 ce=2.7460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.95433812122792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002328201662749052, 'last_eos_top1': 0} +step=81950 lr=6.000000e-04 loss=2.8003 ce=2.8003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1055351933464408e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023208463098853827, 'last_eos_top1': 0} +step=82000 lr=6.000000e-04 loss=2.9272 ce=2.9272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.508281901711598e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015087570063769817, 'last_eos_top1': 0} +step=82050 lr=6.000000e-04 loss=2.6294 ce=2.6294 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9274810913193505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012471445370465517, 'last_eos_top1': 0} +step=82100 lr=6.000000e-04 loss=2.1415 ce=2.1415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4169865911244415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002977306256070733, 'last_eos_top1': 0} +step=82150 lr=6.000000e-04 loss=2.8465 ce=2.8465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3698051589017268e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00306624174118042, 'last_eos_top1': 0} +step=82200 lr=6.000000e-04 loss=2.6399 ce=2.6399 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2628752301679924e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002644615713506937, 'last_eos_top1': 0} +step=82250 lr=6.000000e-04 loss=2.3824 ce=2.3824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.085530766431475e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002160117495805025, 'last_eos_top1': 0} +step=82300 lr=6.000000e-04 loss=2.4229 ce=2.4229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2349677490419708e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003463250817731023, 'last_eos_top1': 0} +step=82350 lr=6.000000e-04 loss=2.4663 ce=2.4663 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.601882468094118e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031124481465667486, 'last_eos_top1': 0} +step=82400 lr=6.000000e-04 loss=2.5266 ce=2.5266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1423588148318231e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003215197240933776, 'last_eos_top1': 0} +step=82450 lr=6.000000e-04 loss=2.5823 ce=2.5823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5945713787223212e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024626203812658787, 'last_eos_top1': 0} +step=82500 lr=6.000000e-04 loss=2.6988 ce=2.6988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3595970560563728e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003818053752183914, 'last_eos_top1': 0} +step=82550 lr=6.000000e-04 loss=2.1525 ce=2.1525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.890812750905752e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004599694162607193, 'last_eos_top1': 0} +step=82600 lr=6.000000e-04 loss=2.4821 ce=2.4821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4252689652494155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033557037822902203, 'last_eos_top1': 0} +step=82650 lr=6.000000e-04 loss=2.5962 ce=2.5962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.983100997284055e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019720476120710373, 'last_eos_top1': 0} +step=82700 lr=6.000000e-04 loss=2.3447 ce=2.3447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.732147317554336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001801256905309856, 'last_eos_top1': 0} +step=82750 lr=6.000000e-04 loss=2.8174 ce=2.8174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.797287253197283e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015446576289832592, 'last_eos_top1': 0} +step=82800 lr=6.000000e-04 loss=2.6652 ce=2.6652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.174555088975467e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021651049610227346, 'last_eos_top1': 0} +step=82850 lr=6.000000e-04 loss=2.4462 ce=2.4462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8556944269221276e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021160945761948824, 'last_eos_top1': 0} +step=82900 lr=6.000000e-04 loss=2.3187 ce=2.3187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6302121366607025e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018370007164776325, 'last_eos_top1': 0} +step=82950 lr=6.000000e-04 loss=2.6514 ce=2.6514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1021415957657155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013517002807930112, 'last_eos_top1': 0} +step=83000 lr=6.000000e-04 loss=2.3946 ce=2.3946 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.671392839052714e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012321188114583492, 'last_eos_top1': 0} +step=83050 lr=6.000000e-04 loss=2.7033 ce=2.7033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.588191015704069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013978455681353807, 'last_eos_top1': 0} +step=83100 lr=6.000000e-04 loss=2.4944 ce=2.4944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.255128881894052e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010177484946325421, 'last_eos_top1': 0} +step=83150 lr=6.000000e-04 loss=2.6736 ce=2.6736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2696973499259911e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002007518894970417, 'last_eos_top1': 0} +step=83200 lr=6.000000e-04 loss=2.3466 ce=2.3466 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3272627256810665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019630605820566416, 'last_eos_top1': 0} +step=83250 lr=6.000000e-04 loss=2.1536 ce=2.1536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.363766366732307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002467832062393427, 'last_eos_top1': 0} +step=83300 lr=6.000000e-04 loss=2.4981 ce=2.4981 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5598117897752672e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003443849738687277, 'last_eos_top1': 0} +step=83350 lr=6.000000e-04 loss=2.5682 ce=2.5682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.278346826438792e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024982765316963196, 'last_eos_top1': 0} +step=83400 lr=6.000000e-04 loss=2.8886 ce=2.8886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7371141186449677e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00226394459605217, 'last_eos_top1': 0} +step=83450 lr=6.000000e-04 loss=2.5239 ce=2.5239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5338549928856082e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002033303491771221, 'last_eos_top1': 0} +step=83500 lr=6.000000e-04 loss=2.3640 ce=2.3640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.80095319391694e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024537495337426662, 'last_eos_top1': 0} +step=83550 lr=6.000000e-04 loss=2.8552 ce=2.8552 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3366493476496544e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002371588721871376, 'last_eos_top1': 0} +step=83600 lr=6.000000e-04 loss=2.3099 ce=2.3099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8928600184153765e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028929526451975107, 'last_eos_top1': 0} +step=83650 lr=6.000000e-04 loss=2.3047 ce=2.3047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5529756637988612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032158996909856796, 'last_eos_top1': 0} +step=83700 lr=6.000000e-04 loss=2.5637 ce=2.5637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3158925361931324e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029522008262574673, 'last_eos_top1': 0} +step=83750 lr=6.000000e-04 loss=2.4568 ce=2.4568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.484730758733349e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024943521711975336, 'last_eos_top1': 0} +step=83800 lr=6.000000e-04 loss=2.2766 ce=2.2766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1243715562159196e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018686664989218116, 'last_eos_top1': 0} +step=83850 lr=6.000000e-04 loss=2.3818 ce=2.3818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4008548532729037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013785955961793661, 'last_eos_top1': 0} +step=83900 lr=6.000000e-04 loss=2.4227 ce=2.4227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.88018257683143e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001378132263198495, 'last_eos_top1': 0} +step=83950 lr=6.000000e-04 loss=2.4826 ce=2.4826 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5622392058721744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016986505361273885, 'last_eos_top1': 0} +step=84000 lr=6.000000e-04 loss=2.3266 ce=2.3266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.672360667726025e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001333424006588757, 'last_eos_top1': 0} +step=84050 lr=6.000000e-04 loss=2.5194 ce=2.5194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.056326467136387e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022037839516997337, 'last_eos_top1': 0} +step=84100 lr=6.000000e-04 loss=2.8292 ce=2.8292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4738366189703811e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019903923384845257, 'last_eos_top1': 0} +step=84150 lr=6.000000e-04 loss=2.9721 ce=2.9721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6502875951118767e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024573917035013437, 'last_eos_top1': 0} +step=84200 lr=6.000000e-04 loss=2.9780 ce=2.9780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7590362404007465e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018787910230457783, 'last_eos_top1': 0} +step=84250 lr=6.000000e-04 loss=2.7376 ce=2.7376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4414772522286512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003589100670069456, 'last_eos_top1': 0} +step=84300 lr=6.000000e-04 loss=2.7263 ce=2.7263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9876417354680598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003206874942407012, 'last_eos_top1': 0} +step=84350 lr=6.000000e-04 loss=2.7600 ce=2.7600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.282785867981147e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002041236963123083, 'last_eos_top1': 0} +step=84400 lr=6.000000e-04 loss=2.3349 ce=2.3349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1963835277128965e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027981437742710114, 'last_eos_top1': 0} +step=84450 lr=6.000000e-04 loss=2.8921 ce=2.8921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2611869351530913e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002715856535360217, 'last_eos_top1': 0} +step=84500 lr=6.000000e-04 loss=2.4127 ce=2.4127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.76650719367899e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032210710924118757, 'last_eos_top1': 0} +step=84550 lr=6.000000e-04 loss=2.5665 ce=2.5665 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.338957528991159e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002726049395278096, 'last_eos_top1': 0} +step=84600 lr=6.000000e-04 loss=2.2348 ce=2.2348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4997774997027591e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00315159372985363, 'last_eos_top1': 0} +step=84650 lr=6.000000e-04 loss=2.3562 ce=2.3562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7821443179855123e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001863175304606557, 'last_eos_top1': 0} +step=84700 lr=6.000000e-04 loss=2.7088 ce=2.7088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.628140489629004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002899517072364688, 'last_eos_top1': 0} +step=84750 lr=6.000000e-04 loss=3.1236 ce=3.1236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4228067811927758e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016976483166217804, 'last_eos_top1': 0} +step=84800 lr=6.000000e-04 loss=2.5678 ce=2.5678 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.509063622506801e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013279472477734089, 'last_eos_top1': 0} +step=84850 lr=6.000000e-04 loss=2.4737 ce=2.4737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6006870029959828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001459990395233035, 'last_eos_top1': 0} +step=84900 lr=6.000000e-04 loss=2.2354 ce=2.2354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.450322977500036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009292610920965672, 'last_eos_top1': 0} +step=84950 lr=6.000000e-04 loss=1.9163 ce=1.9163 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6670150216668844e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010085395770147443, 'last_eos_top1': 0} +step=85000 lr=6.000000e-04 loss=2.2021 ce=2.2021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.076858648564667e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011763502843677998, 'last_eos_top1': 0} +step=85050 lr=6.000000e-04 loss=2.8262 ce=2.8262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5757008441141807e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001153237884864211, 'last_eos_top1': 0} +step=85100 lr=6.000000e-04 loss=2.2969 ce=2.2969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9020133549929596e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001407178700901568, 'last_eos_top1': 0} +step=85150 lr=6.000000e-04 loss=2.4164 ce=2.4164 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2090940799680538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015396703965961933, 'last_eos_top1': 0} +step=85200 lr=6.000000e-04 loss=2.3625 ce=2.3625 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.706998773443047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00112791801802814, 'last_eos_top1': 0} +step=85250 lr=6.000000e-04 loss=3.1807 ce=3.1807 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4509502761939075e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023652210365980864, 'last_eos_top1': 0} +step=85300 lr=6.000000e-04 loss=1.7304 ce=1.7304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.983874972211197e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022153041791170835, 'last_eos_top1': 0} +step=85350 lr=6.000000e-04 loss=2.9644 ce=2.9644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.44360037666047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024552622344344854, 'last_eos_top1': 0} +step=85400 lr=6.000000e-04 loss=2.3063 ce=2.3063 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.620567546633538e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002500147558748722, 'last_eos_top1': 0} +step=85450 lr=6.000000e-04 loss=3.1326 ce=3.1326 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0902461983496323e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008813516469672322, 'last_eos_top1': 0} +step=85500 lr=6.000000e-04 loss=2.4550 ce=2.4550 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.896548565127887e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014000581577420235, 'last_eos_top1': 0} +step=85550 lr=6.000000e-04 loss=3.0370 ce=3.0370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2839855116908439e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012065095361322165, 'last_eos_top1': 0} +step=85600 lr=6.000000e-04 loss=2.8422 ce=2.8422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.770880408002995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001927653793245554, 'last_eos_top1': 0} +step=85650 lr=6.000000e-04 loss=2.5478 ce=2.5478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.466000139771495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016394542763009667, 'last_eos_top1': 0} +step=85700 lr=6.000000e-04 loss=2.0879 ce=2.0879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2847054676967673e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012651984579861164, 'last_eos_top1': 0} +step=85750 lr=6.000000e-04 loss=2.0611 ce=2.0611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9925186279579066e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015069362707436085, 'last_eos_top1': 0} +step=85800 lr=6.000000e-04 loss=2.6033 ce=2.6033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3666188351635356e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011185270268470049, 'last_eos_top1': 0} +step=85850 lr=6.000000e-04 loss=2.4370 ce=2.4370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.548695637320634e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009911764645949006, 'last_eos_top1': 0} +step=85900 lr=6.000000e-04 loss=2.7518 ce=2.7518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.615909347194247e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013187569566071033, 'last_eos_top1': 0} +step=85950 lr=6.000000e-04 loss=2.6494 ce=2.6494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.281004061747808e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00102553004398942, 'last_eos_top1': 0} +step=86000 lr=6.000000e-04 loss=2.7593 ce=2.7593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.861629223451018e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002163754543289542, 'last_eos_top1': 0} +step=86050 lr=6.000000e-04 loss=2.8246 ce=2.8246 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.421351827157196e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001829138956964016, 'last_eos_top1': 0} +step=86100 lr=6.000000e-04 loss=2.8543 ce=2.8543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4517107047140598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020178728736937046, 'last_eos_top1': 0} +step=86150 lr=6.000000e-04 loss=2.2054 ce=2.2054 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9759088900173083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002185746096074581, 'last_eos_top1': 0} +step=86200 lr=6.000000e-04 loss=2.8270 ce=2.8270 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7601860236027278e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015944638289511204, 'last_eos_top1': 0} +step=86250 lr=6.000000e-04 loss=2.6476 ce=2.6476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4377366824191995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002940963953733444, 'last_eos_top1': 0} +step=86300 lr=6.000000e-04 loss=2.5369 ce=2.5369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.460915336792823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029576257802546024, 'last_eos_top1': 0} +step=86350 lr=6.000000e-04 loss=2.6219 ce=2.6219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.804860615578946e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002658066339790821, 'last_eos_top1': 0} +step=86400 lr=6.000000e-04 loss=2.4870 ce=2.4870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3825287977815606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010692602954804897, 'last_eos_top1': 0} +step=86450 lr=6.000000e-04 loss=2.4336 ce=2.4336 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.309761258511571e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005760174244642258, 'last_eos_top1': 0} +step=86500 lr=6.000000e-04 loss=2.3789 ce=2.3789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0705805834732018e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011985143646597862, 'last_eos_top1': 0} +step=86550 lr=6.000000e-04 loss=2.3049 ce=2.3049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5958602135651745e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012699775397777557, 'last_eos_top1': 0} +step=86600 lr=6.000000e-04 loss=2.5987 ce=2.5987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1474687198642641e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017411817098036408, 'last_eos_top1': 0} +step=86650 lr=6.000000e-04 loss=2.6424 ce=2.6424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1593948986264877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015430146595463157, 'last_eos_top1': 0} +step=86700 lr=6.000000e-04 loss=2.3103 ce=2.3103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.24081025205669e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015719094080850482, 'last_eos_top1': 0} +step=86750 lr=6.000000e-04 loss=2.6104 ce=2.6104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.1100031416572165e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007775338599458337, 'last_eos_top1': 0} +step=86800 lr=6.000000e-04 loss=2.8283 ce=2.8283 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0729190762504004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021418300457298756, 'last_eos_top1': 0} +step=86850 lr=6.000000e-04 loss=2.3770 ce=2.3770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.25490429229103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017411187291145325, 'last_eos_top1': 0} +step=86900 lr=6.000000e-04 loss=2.3650 ce=2.3650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.328839218534995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014877611538395286, 'last_eos_top1': 0} +step=86950 lr=6.000000e-04 loss=2.6431 ce=2.6431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.007150422083214e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011286204680800438, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1400000 rows=1391809 buffered=8191 part=openwebtext-t5 +step=87000 lr=6.000000e-04 loss=2.7673 ce=2.7673 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3896699783799704e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001407491392455995, 'last_eos_top1': 0} +step=87050 lr=6.000000e-04 loss=2.6448 ce=2.6448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.549631613481324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032282532192766666, 'last_eos_top1': 0} +step=87100 lr=6.000000e-04 loss=2.3228 ce=2.3228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.551372957299463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035241201985627413, 'last_eos_top1': 0} +step=87150 lr=6.000000e-04 loss=1.9490 ce=1.9490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3776482774119359e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022417609579861164, 'last_eos_top1': 0} +step=87200 lr=6.000000e-04 loss=2.3005 ce=2.3005 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.47072158951778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030140415765345097, 'last_eos_top1': 0} +step=87250 lr=6.000000e-04 loss=2.4909 ce=2.4909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3616370526724495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015318747609853745, 'last_eos_top1': 0} +step=87300 lr=6.000000e-04 loss=2.6438 ce=2.6438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.442394725221675e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014791747089475393, 'last_eos_top1': 0} +step=87350 lr=6.000000e-04 loss=2.6050 ce=2.6050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.007596418086905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026139914989471436, 'last_eos_top1': 0} +step=87400 lr=6.000000e-04 loss=2.8641 ce=2.8641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0484294762136415e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001980710308998823, 'last_eos_top1': 0} +step=87450 lr=6.000000e-04 loss=2.2725 ce=2.2725 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.331317076226696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001985954586416483, 'last_eos_top1': 0} +step=87500 lr=6.000000e-04 loss=2.6987 ce=2.6987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.870574568689335e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035008511040359735, 'last_eos_top1': 0} +step=87550 lr=6.000000e-04 loss=2.3382 ce=2.3382 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8888447812059894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023059011436998844, 'last_eos_top1': 0} +step=87600 lr=6.000000e-04 loss=3.0880 ce=3.0880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.228776091011241e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022610239684581757, 'last_eos_top1': 0} +step=87650 lr=6.000000e-04 loss=2.1123 ce=2.1123 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8758959059778135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001012985478155315, 'last_eos_top1': 0} +step=87700 lr=6.000000e-04 loss=2.4425 ce=2.4425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.989589575008722e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001689353957772255, 'last_eos_top1': 0} +step=87750 lr=6.000000e-04 loss=2.5968 ce=2.5968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.87218278774526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022369823418557644, 'last_eos_top1': 0} +step=87800 lr=6.000000e-04 loss=2.5086 ce=2.5086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0181538527831435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002061841543763876, 'last_eos_top1': 0} +step=87850 lr=6.000000e-04 loss=2.7523 ce=2.7523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.289386303455103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002729254076257348, 'last_eos_top1': 0} +step=87900 lr=6.000000e-04 loss=2.2346 ce=2.2346 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.007958300964674e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019353668903931975, 'last_eos_top1': 0} +step=87950 lr=6.000000e-04 loss=2.6215 ce=2.6215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.979462447518017e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001956445397809148, 'last_eos_top1': 0} +step=88000 lr=6.000000e-04 loss=2.7029 ce=2.7029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.350664873229107e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002572921570390463, 'last_eos_top1': 0} +step=88050 lr=6.000000e-04 loss=3.0409 ce=3.0409 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.832066669128835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028963973745703697, 'last_eos_top1': 0} +step=88100 lr=6.000000e-04 loss=2.7006 ce=2.7006 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8460768741497304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027348287403583527, 'last_eos_top1': 0} +step=88150 lr=6.000000e-04 loss=2.8544 ce=2.8544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.329072781838477e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003424534108489752, 'last_eos_top1': 0} +step=88200 lr=6.000000e-04 loss=2.8751 ce=2.8751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.12342477729544e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038304512854665518, 'last_eos_top1': 0} +step=88250 lr=6.000000e-04 loss=2.3383 ce=2.3383 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3755665349890478e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003183854278177023, 'last_eos_top1': 0} +step=88300 lr=6.000000e-04 loss=2.7168 ce=2.7168 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3196391591918655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002567548770457506, 'last_eos_top1': 0} +step=88350 lr=6.000000e-04 loss=1.9237 ce=1.9237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.832077032304369e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005465202033519745, 'last_eos_top1': 0} +step=88400 lr=6.000000e-04 loss=3.0739 ce=3.0739 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1395742805907503e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030848379246890545, 'last_eos_top1': 0} +step=88450 lr=6.000000e-04 loss=2.3327 ce=2.3327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7704594938550144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030701104551553726, 'last_eos_top1': 0} +step=88500 lr=6.000000e-04 loss=2.1803 ce=2.1803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6502937796758488e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004571533761918545, 'last_eos_top1': 0} +step=88550 lr=6.000000e-04 loss=1.9649 ce=1.9649 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.829009412787855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034375907853245735, 'last_eos_top1': 0} +step=88600 lr=6.000000e-04 loss=2.5185 ce=2.5185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.845973307965323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00199397886171937, 'last_eos_top1': 0} +step=88650 lr=6.000000e-04 loss=2.5748 ce=2.5748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.815767609514296e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001202740939334035, 'last_eos_top1': 0} +step=88700 lr=6.000000e-04 loss=2.4989 ce=2.4989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3674132787855342e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012398194521665573, 'last_eos_top1': 0} +step=88750 lr=6.000000e-04 loss=2.8667 ce=2.8667 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.736311060085427e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016670720651745796, 'last_eos_top1': 0} +step=88800 lr=6.000000e-04 loss=2.3462 ce=2.3462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1434640327934176e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008849342120811343, 'last_eos_top1': 0} +step=88850 lr=6.000000e-04 loss=2.7112 ce=2.7112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.019379154371563e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003154802543576807, 'last_eos_top1': 0} +step=88900 lr=6.000000e-04 loss=2.4608 ce=2.4608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.117575245909393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003379129630047828, 'last_eos_top1': 0} +step=88950 lr=6.000000e-04 loss=2.2457 ce=2.2457 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.349002468487015e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008807713747955859, 'last_eos_top1': 0} +step=89000 lr=6.000000e-04 loss=2.5302 ce=2.5302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3223890821100213e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004555257910396904, 'last_eos_top1': 0} +step=89050 lr=6.000000e-04 loss=1.7348 ce=1.7348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7219066648976877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003670096048153937, 'last_eos_top1': 0} +step=89100 lr=6.000000e-04 loss=2.5640 ce=2.5640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7605201466940343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005522056599147618, 'last_eos_top1': 0} +step=89150 lr=6.000000e-04 loss=2.6956 ce=2.6956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.824661376303993e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002898589300457388, 'last_eos_top1': 0} +step=89200 lr=6.000000e-04 loss=2.3287 ce=2.3287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.281204256694764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022521504433825612, 'last_eos_top1': 0} +step=89250 lr=6.000000e-04 loss=2.7106 ce=2.7106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.826657333178446e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038533745100721717, 'last_eos_top1': 0} +step=89300 lr=6.000000e-04 loss=2.6940 ce=2.6940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.481574589794036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021628713875543326, 'last_eos_top1': 0} +step=89350 lr=6.000000e-04 loss=2.0537 ce=2.0537 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.205118097568629e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001935871405294165, 'last_eos_top1': 0} +step=89400 lr=6.000000e-04 loss=2.8160 ce=2.8160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0064763046102598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030681880889460444, 'last_eos_top1': 0} +step=89450 lr=6.000000e-04 loss=2.4025 ce=2.4025 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.409092515648808e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004575400671456009, 'last_eos_top1': 0} +step=89500 lr=6.000000e-04 loss=2.8683 ce=2.8683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.648146462684963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006409474881365895, 'last_eos_top1': 0} +step=89550 lr=6.000000e-04 loss=2.0883 ce=2.0883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.568461059127003e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008720242767594755, 'last_eos_top1': 0} +step=89600 lr=6.000000e-04 loss=2.6903 ce=2.6903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1478203305159695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010919044725596905, 'last_eos_top1': 0} +step=89650 lr=6.000000e-04 loss=2.1251 ce=2.1251 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.047375892288983e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00168694113381207, 'last_eos_top1': 0} +step=89700 lr=6.000000e-04 loss=2.5773 ce=2.5773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.393520147365052e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014177324483171105, 'last_eos_top1': 0} +step=89750 lr=6.000000e-04 loss=2.3379 ce=2.3379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.936038622981869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005140460561960936, 'last_eos_top1': 0} +step=89800 lr=6.000000e-04 loss=2.3778 ce=2.3778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.754253601888195e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011191063094884157, 'last_eos_top1': 0} +step=89850 lr=6.000000e-04 loss=1.8379 ce=1.8379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5358215023297817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016059796325862408, 'last_eos_top1': 0} +step=89900 lr=6.000000e-04 loss=2.4034 ce=2.4034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0189298639888875e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018300465308129787, 'last_eos_top1': 0} +step=89950 lr=6.000000e-04 loss=2.5541 ce=2.5541 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.641163731226698e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033980561420321465, 'last_eos_top1': 0} +step=90000 lr=6.000000e-04 loss=2.6705 ce=2.6705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.947001798427664e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005190824158489704, 'last_eos_top1': 0} +step=90050 lr=6.000000e-04 loss=2.5139 ce=2.5139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.432228474295698e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003909857477992773, 'last_eos_top1': 0} +step=90100 lr=6.000000e-04 loss=2.2327 ce=2.2327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7437927454011515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026927166618406773, 'last_eos_top1': 0} +step=90150 lr=6.000000e-04 loss=2.7534 ce=2.7534 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.903052392066456e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004233534447848797, 'last_eos_top1': 0} +step=90200 lr=6.000000e-04 loss=2.8306 ce=2.8306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.792050498916069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004626140929758549, 'last_eos_top1': 0} +step=90250 lr=6.000000e-04 loss=2.8733 ce=2.8733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.065217687340919e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017756244633346796, 'last_eos_top1': 0} +step=90300 lr=6.000000e-04 loss=2.4088 ce=2.4088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.091553248348646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015543743502348661, 'last_eos_top1': 0} +step=90350 lr=6.000000e-04 loss=2.4582 ce=2.4582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.92181482311571e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033678440377116203, 'last_eos_top1': 0} +step=90400 lr=6.000000e-04 loss=2.7911 ce=2.7911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.885606803232804e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019519008928909898, 'last_eos_top1': 0} +step=90450 lr=6.000000e-04 loss=2.7290 ce=2.7290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.469655323191546e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014364535454660654, 'last_eos_top1': 0} +step=90500 lr=6.000000e-04 loss=2.9527 ce=2.9527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.953887456504162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015661363722756505, 'last_eos_top1': 0} +step=90550 lr=6.000000e-04 loss=1.9358 ce=1.9358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4501904337957967e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008267976809293032, 'last_eos_top1': 0} +step=90600 lr=6.000000e-04 loss=2.2182 ce=2.2182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.682313258148497e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016981333028525114, 'last_eos_top1': 0} +step=90650 lr=6.000000e-04 loss=2.5042 ce=2.5042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6795792008633725e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021698842756450176, 'last_eos_top1': 0} +step=90700 lr=6.000000e-04 loss=3.0500 ce=3.0500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.241467907675542e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027591860853135586, 'last_eos_top1': 0} +step=90750 lr=6.000000e-04 loss=2.5717 ce=2.5717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.730480087455362e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002899754326790571, 'last_eos_top1': 0} +step=90800 lr=6.000000e-04 loss=2.5835 ce=2.5835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0357300197938457e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003109400626271963, 'last_eos_top1': 0} +step=90850 lr=6.000000e-04 loss=2.2990 ce=2.2990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.838611625425983e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002683308906853199, 'last_eos_top1': 0} +step=90900 lr=6.000000e-04 loss=3.0188 ce=3.0188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0035519153461792e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009545165812596679, 'last_eos_top1': 0} +step=90950 lr=6.000000e-04 loss=2.1705 ce=2.1705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.585853952856269e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023130811750888824, 'last_eos_top1': 0} +step=91000 lr=6.000000e-04 loss=2.5523 ce=2.5523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.05451418273151e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031954683363437653, 'last_eos_top1': 0} +step=91050 lr=6.000000e-04 loss=2.5161 ce=2.5161 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3546708942158148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018366080475971103, 'last_eos_top1': 0} +step=91100 lr=6.000000e-04 loss=2.5482 ce=2.5482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1225845810258761e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006381495040841401, 'last_eos_top1': 0} +step=91150 lr=6.000000e-04 loss=2.4336 ce=2.4336 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.767583494773135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021708598360419273, 'last_eos_top1': 0} +step=91200 lr=6.000000e-04 loss=2.6821 ce=2.6821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9205955292563885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004426578059792519, 'last_eos_top1': 0} +step=91250 lr=6.000000e-04 loss=1.8332 ce=1.8332 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.486863301484846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037317732349038124, 'last_eos_top1': 0} +step=91300 lr=6.000000e-04 loss=2.8930 ce=2.8930 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.750451914034784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028918571770191193, 'last_eos_top1': 0} +step=91350 lr=6.000000e-04 loss=2.6653 ce=2.6653 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.745926268398762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029001752845942974, 'last_eos_top1': 0} +step=91400 lr=6.000000e-04 loss=2.6922 ce=2.6922 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.557295925333165e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017293139826506376, 'last_eos_top1': 0} +step=91450 lr=6.000000e-04 loss=2.8698 ce=2.8698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3922187463322189e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002948298119008541, 'last_eos_top1': 0} +step=91500 lr=6.000000e-04 loss=2.2933 ce=2.2933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2703305401373655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003340568393468857, 'last_eos_top1': 0} +step=91550 lr=6.000000e-04 loss=2.8864 ce=2.8864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1310727131785825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026536842342466116, 'last_eos_top1': 0} +step=91600 lr=6.000000e-04 loss=2.3263 ce=2.3263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7665408449829556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036120698787271976, 'last_eos_top1': 0} +step=91650 lr=6.000000e-04 loss=2.4783 ce=2.4783 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5952315304020885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025811661034822464, 'last_eos_top1': 0} +step=91700 lr=6.000000e-04 loss=2.9219 ce=2.9219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.02950341696851e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016500013880431652, 'last_eos_top1': 0} +step=91750 lr=6.000000e-04 loss=2.4032 ce=2.4032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.059722414240241e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018384482245892286, 'last_eos_top1': 0} +step=91800 lr=6.000000e-04 loss=2.5798 ce=2.5798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.61518266092753e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002076196251437068, 'last_eos_top1': 0} +step=91850 lr=6.000000e-04 loss=2.9905 ce=2.9905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5402933058794588e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002604844281449914, 'last_eos_top1': 0} +step=91900 lr=6.000000e-04 loss=2.7700 ce=2.7700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.415887244453188e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023094757925719023, 'last_eos_top1': 0} +step=91950 lr=6.000000e-04 loss=2.8874 ce=2.8874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.649889772030292e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033977448474615812, 'last_eos_top1': 0} +step=92000 lr=6.000000e-04 loss=2.9399 ce=2.9399 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0202632438449655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026065572164952755, 'last_eos_top1': 0} +step=92050 lr=6.000000e-04 loss=2.5864 ce=2.5864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.601489192486042e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032446696422994137, 'last_eos_top1': 0} +step=92100 lr=6.000000e-04 loss=3.0491 ce=3.0491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.670658305869438e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036833640187978745, 'last_eos_top1': 0} +step=92150 lr=6.000000e-04 loss=2.7965 ce=2.7965 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9519467261707177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033982284367084503, 'last_eos_top1': 0} +step=92200 lr=6.000000e-04 loss=2.1883 ce=2.1883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1159359019075055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033023152500391006, 'last_eos_top1': 0} +step=92250 lr=6.000000e-04 loss=2.5190 ce=2.5190 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.542575773783028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037366687320172787, 'last_eos_top1': 0} +step=92300 lr=6.000000e-04 loss=2.6649 ce=2.6649 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.072306521469727e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033102678135037422, 'last_eos_top1': 0} +step=92350 lr=6.000000e-04 loss=2.5182 ce=2.5182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0912914149230346e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005121096037328243, 'last_eos_top1': 0} +step=92400 lr=6.000000e-04 loss=2.6253 ce=2.6253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.631369615206495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005079696886241436, 'last_eos_top1': 0} +step=92450 lr=6.000000e-04 loss=2.6905 ce=2.6905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.726779585937038e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004437996074557304, 'last_eos_top1': 0} +step=92500 lr=6.000000e-04 loss=2.3177 ce=2.3177 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.474648787640035e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003592248074710369, 'last_eos_top1': 0} +step=92550 lr=6.000000e-04 loss=2.5139 ce=2.5139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0095499065937474e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003953995648771524, 'last_eos_top1': 0} +step=92600 lr=6.000000e-04 loss=2.5794 ce=2.5794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1519189683895092e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003973729908466339, 'last_eos_top1': 0} +step=92650 lr=6.000000e-04 loss=2.9052 ce=2.9052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.889118933235295e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004101731814444065, 'last_eos_top1': 0} +step=92700 lr=6.000000e-04 loss=2.2356 ce=2.2356 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.69718099036254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00368714053183794, 'last_eos_top1': 0} +step=92750 lr=6.000000e-04 loss=2.9295 ce=2.9295 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1351968320668675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002989640925079584, 'last_eos_top1': 0} +step=92800 lr=6.000000e-04 loss=1.9163 ce=1.9163 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4344875125971157e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004244748502969742, 'last_eos_top1': 0} +step=92850 lr=6.000000e-04 loss=2.1284 ce=2.1284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.306196155492216e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028053116984665394, 'last_eos_top1': 0} +step=92900 lr=6.000000e-04 loss=2.3400 ce=2.3400 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.44842907099519e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025764426682144403, 'last_eos_top1': 0} +step=92950 lr=6.000000e-04 loss=2.8949 ce=2.8949 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5453532442916185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032620010897517204, 'last_eos_top1': 0} +step=93000 lr=6.000000e-04 loss=2.2569 ce=2.2569 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.234826393367257e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020219075959175825, 'last_eos_top1': 0} +step=93050 lr=6.000000e-04 loss=3.0568 ce=3.0568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.411019745020894e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00237381923943758, 'last_eos_top1': 0} +step=93100 lr=6.000000e-04 loss=2.3652 ce=2.3652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.712614438380115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028189322911202908, 'last_eos_top1': 0} +step=93150 lr=6.000000e-04 loss=3.3024 ce=3.3024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.155229246156523e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002400434110313654, 'last_eos_top1': 0} +step=93200 lr=6.000000e-04 loss=2.6483 ce=2.6483 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.461495336727239e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003993580117821693, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1500000 rows=1491809 buffered=8191 part=openwebtext-t5 +step=93250 lr=6.000000e-04 loss=2.8095 ce=2.8095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.269440407573711e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038931332528591156, 'last_eos_top1': 0} +step=93300 lr=6.000000e-04 loss=2.9817 ce=2.9817 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.033895831729751e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003130086697638035, 'last_eos_top1': 0} +step=93350 lr=6.000000e-04 loss=3.0617 ce=3.0617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.112651251693023e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002623223001137376, 'last_eos_top1': 0} +step=93400 lr=6.000000e-04 loss=2.6278 ce=2.6278 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0985204426106066e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004462442360818386, 'last_eos_top1': 0} +step=93450 lr=6.000000e-04 loss=2.1586 ce=2.1586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.92318428971339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029075315687805414, 'last_eos_top1': 0} +step=93500 lr=6.000000e-04 loss=2.8008 ce=2.8008 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.986521268321667e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002399963326752186, 'last_eos_top1': 0} +step=93550 lr=6.000000e-04 loss=2.4017 ce=2.4017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.115440555731766e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001071631209924817, 'last_eos_top1': 0} +step=93600 lr=6.000000e-04 loss=2.9933 ce=2.9933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.678268735209713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012815350200980902, 'last_eos_top1': 0} +step=93650 lr=6.000000e-04 loss=2.4599 ce=2.4599 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.24698542803526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017407184932380915, 'last_eos_top1': 0} +step=93700 lr=6.000000e-04 loss=2.5875 ce=2.5875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.512427297711838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023625718895345926, 'last_eos_top1': 0} +step=93750 lr=6.000000e-04 loss=2.6408 ce=2.6408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.88563876185799e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008709343965165317, 'last_eos_top1': 0} +step=93800 lr=6.000000e-04 loss=2.7608 ce=2.7608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.137865395634435e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014226255007088184, 'last_eos_top1': 0} +step=93850 lr=6.000000e-04 loss=2.9697 ce=2.9697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8610769479419105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008310086559504271, 'last_eos_top1': 0} +step=93900 lr=6.000000e-04 loss=3.1222 ce=3.1222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7036561479908414e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002091383095830679, 'last_eos_top1': 0} +step=93950 lr=6.000000e-04 loss=2.6834 ce=2.6834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3335278708836995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002221031580120325, 'last_eos_top1': 0} +step=94000 lr=6.000000e-04 loss=2.6864 ce=2.6864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.022674041450955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001169928815215826, 'last_eos_top1': 0} +step=94050 lr=6.000000e-04 loss=2.6147 ce=2.6147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0711467439250555e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011871682945638895, 'last_eos_top1': 0} +step=94100 lr=6.000000e-04 loss=2.2190 ce=2.2190 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0326170013286173e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001037738285958767, 'last_eos_top1': 0} +step=94150 lr=6.000000e-04 loss=2.4316 ce=2.4316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.2867121566087e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002333966549485922, 'last_eos_top1': 0} +step=94200 lr=6.000000e-04 loss=2.5342 ce=2.5342 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0661902706488036e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021540867164731026, 'last_eos_top1': 0} +step=94250 lr=6.000000e-04 loss=2.2440 ce=2.2440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.925581601535669e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001414483180269599, 'last_eos_top1': 0} +step=94300 lr=6.000000e-04 loss=2.8203 ce=2.8203 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6732590160681866e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009893765673041344, 'last_eos_top1': 0} +step=94350 lr=6.000000e-04 loss=2.2718 ce=2.2718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.528744623006787e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018516471609473228, 'last_eos_top1': 0} +step=94400 lr=6.000000e-04 loss=2.9607 ce=2.9607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.916956747067161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027060043066740036, 'last_eos_top1': 0} +step=94450 lr=6.000000e-04 loss=1.9174 ce=1.9174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.085319793986855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001990916207432747, 'last_eos_top1': 0} +step=94500 lr=6.000000e-04 loss=2.3774 ce=2.3774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.516070349491201e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033665294758975506, 'last_eos_top1': 0} +step=94550 lr=6.000000e-04 loss=2.2040 ce=2.2040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.73845897836145e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002224341966211796, 'last_eos_top1': 0} +step=94600 lr=6.000000e-04 loss=2.9111 ce=2.9111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2302993127377704e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008624392794445157, 'last_eos_top1': 0} +step=94650 lr=6.000000e-04 loss=2.5476 ce=2.5476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.194000469695311e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016620759852230549, 'last_eos_top1': 0} +step=94700 lr=6.000000e-04 loss=2.3209 ce=2.3209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.406112642143853e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012647375697270036, 'last_eos_top1': 0} +step=94750 lr=6.000000e-04 loss=1.9399 ce=1.9399 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.330817192676477e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014664286281913519, 'last_eos_top1': 0} +step=94800 lr=6.000000e-04 loss=2.2435 ce=2.2435 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.67821597441798e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010120593942701817, 'last_eos_top1': 0} +step=94850 lr=6.000000e-04 loss=2.5980 ce=2.5980 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0367661161581054e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011268823873251677, 'last_eos_top1': 0} +step=94900 lr=6.000000e-04 loss=2.9280 ce=2.9280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.011194950086065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001147241797298193, 'last_eos_top1': 0} +step=94950 lr=6.000000e-04 loss=2.2896 ce=2.2896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.44652903045062e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009048323263414204, 'last_eos_top1': 0} +step=95000 lr=6.000000e-04 loss=2.4909 ce=2.4909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.669830665690824e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000821230816654861, 'last_eos_top1': 0} +step=95050 lr=6.000000e-04 loss=2.8114 ce=2.8114 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.774004305043491e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012504952028393745, 'last_eos_top1': 0} +step=95100 lr=6.000000e-04 loss=2.6511 ce=2.6511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.498000635067001e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006213349406607449, 'last_eos_top1': 0} +step=95150 lr=6.000000e-04 loss=2.2795 ce=2.2795 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.103769464971265e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000458297785371542, 'last_eos_top1': 0} +step=95200 lr=6.000000e-04 loss=2.5596 ce=2.5596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.138347089290619e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005742472130805254, 'last_eos_top1': 0} +step=95250 lr=6.000000e-04 loss=2.3308 ce=2.3308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0444588042446412e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011828672140836716, 'last_eos_top1': 0} +step=95300 lr=6.000000e-04 loss=2.5614 ce=2.5614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.430689947796054e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007190264295786619, 'last_eos_top1': 0} +step=95350 lr=6.000000e-04 loss=2.5475 ce=2.5475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.630745024769567e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009563869680278003, 'last_eos_top1': 0} +step=95400 lr=6.000000e-04 loss=2.3360 ce=2.3360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7387536647438537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008232060354202986, 'last_eos_top1': 0} +step=95450 lr=6.000000e-04 loss=2.7992 ce=2.7992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.984981832327321e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005413021426647902, 'last_eos_top1': 0} +step=95500 lr=6.000000e-04 loss=2.1788 ce=2.1788 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.337516995496117e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007092203013598919, 'last_eos_top1': 0} +step=95550 lr=6.000000e-04 loss=2.5607 ce=2.5607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.317173862655181e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038153125206008554, 'last_eos_top1': 0} +step=95600 lr=6.000000e-04 loss=2.6018 ce=2.6018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7822441047173925e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018884934252128005, 'last_eos_top1': 0} +step=95650 lr=6.000000e-04 loss=2.4317 ce=2.4317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7934539654525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000503490911796689, 'last_eos_top1': 0} +step=95700 lr=6.000000e-04 loss=2.3909 ce=2.3909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.484275789058302e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047806964721530676, 'last_eos_top1': 0} +step=95750 lr=6.000000e-04 loss=2.4194 ce=2.4194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1442623872426338e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00041615916416049004, 'last_eos_top1': 0} +step=95800 lr=6.000000e-04 loss=2.6939 ce=2.6939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6511105463432614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006640589563176036, 'last_eos_top1': 0} +step=95850 lr=6.000000e-04 loss=2.8770 ce=2.8770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.563011654885486e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038095959462225437, 'last_eos_top1': 0} +step=95900 lr=6.000000e-04 loss=2.6538 ce=2.6538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.441285000211792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020711871911771595, 'last_eos_top1': 0} +step=95950 lr=6.000000e-04 loss=2.1119 ce=2.1119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.938748578657396e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006159390322864056, 'last_eos_top1': 0} +step=96000 lr=6.000000e-04 loss=2.2402 ce=2.2402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.746795406797901e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000985768623650074, 'last_eos_top1': 0} +step=96050 lr=6.000000e-04 loss=2.4737 ce=2.4737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.686061740561854e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014159542042762041, 'last_eos_top1': 0} +step=96100 lr=6.000000e-04 loss=2.1951 ce=2.1951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.174534635443706e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001005669473670423, 'last_eos_top1': 0} +step=96150 lr=6.000000e-04 loss=2.5360 ce=2.5360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.464414698508335e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006496479036286473, 'last_eos_top1': 0} +step=96200 lr=6.000000e-04 loss=2.9004 ce=2.9004 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6563110420975136e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045050698099657893, 'last_eos_top1': 0} +step=96250 lr=6.000000e-04 loss=2.9372 ce=2.9372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.801754585059825e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007217300590127707, 'last_eos_top1': 0} +step=96300 lr=6.000000e-04 loss=2.8762 ce=2.8762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5876563540805364e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016800044104456902, 'last_eos_top1': 0} +step=96350 lr=6.000000e-04 loss=2.5026 ce=2.5026 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.081627477920847e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007209752220660448, 'last_eos_top1': 0} +step=96400 lr=6.000000e-04 loss=2.8471 ce=2.8471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7954230265313527e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005681770853698254, 'last_eos_top1': 0} +step=96450 lr=6.000000e-04 loss=2.6792 ce=2.6792 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4270083233423065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008922208799049258, 'last_eos_top1': 0} +step=96500 lr=6.000000e-04 loss=2.9062 ce=2.9062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.60938838639413e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001878881361335516, 'last_eos_top1': 0} +step=96550 lr=6.000000e-04 loss=1.9937 ce=1.9937 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.966574462945573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005298588657751679, 'last_eos_top1': 0} +step=96600 lr=6.000000e-04 loss=2.3728 ce=2.3728 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8756783902063034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006444030441343784, 'last_eos_top1': 0} +step=96650 lr=6.000000e-04 loss=2.9947 ce=2.9947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5270132280420512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004092084418516606, 'last_eos_top1': 0} +step=96700 lr=6.000000e-04 loss=2.6034 ce=2.6034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.309933375334367e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005436128121800721, 'last_eos_top1': 0} +step=96750 lr=6.000000e-04 loss=2.5962 ce=2.5962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8996000764891505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001972507598111406, 'last_eos_top1': 0} +step=96800 lr=6.000000e-04 loss=2.8259 ce=2.8259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.897595762391575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019254055805504322, 'last_eos_top1': 0} +step=96850 lr=6.000000e-04 loss=2.5286 ce=2.5286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8439924335543765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009013403905555606, 'last_eos_top1': 0} +step=96900 lr=6.000000e-04 loss=2.2198 ce=2.2198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.228302714182064e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009573312127031386, 'last_eos_top1': 0} +step=96950 lr=6.000000e-04 loss=3.0178 ce=3.0178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.623975761816837e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013609110610559583, 'last_eos_top1': 0} +step=97000 lr=6.000000e-04 loss=2.6776 ce=2.6776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0896725295169745e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000452077598311007, 'last_eos_top1': 0} +step=97050 lr=6.000000e-04 loss=3.0624 ce=3.0624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.070918400131632e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008349105482921004, 'last_eos_top1': 0} +step=97100 lr=6.000000e-04 loss=2.3354 ce=2.3354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.478422387270257e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028739674016833305, 'last_eos_top1': 0} +step=97150 lr=6.000000e-04 loss=2.6223 ce=2.6223 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5822524750983575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002782936906442046, 'last_eos_top1': 0} +step=97200 lr=6.000000e-04 loss=2.3102 ce=2.3102 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.683424205926713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038877353072166443, 'last_eos_top1': 0} +step=97250 lr=6.000000e-04 loss=3.0601 ce=3.0601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5406347857788205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005080851726233959, 'last_eos_top1': 0} +step=97300 lr=6.000000e-04 loss=2.6956 ce=2.6956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5419980122242123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00556337833404541, 'last_eos_top1': 0} +step=97350 lr=6.000000e-04 loss=2.6133 ce=2.6133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.278856744756922e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029444308020174503, 'last_eos_top1': 0} +step=97400 lr=6.000000e-04 loss=2.7869 ce=2.7869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.812684089003596e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018351500621065497, 'last_eos_top1': 0} +step=97450 lr=6.000000e-04 loss=2.7372 ce=2.7372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.586906122014625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028190049342811108, 'last_eos_top1': 0} +step=97500 lr=6.000000e-04 loss=2.7831 ce=2.7831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1821373846032657e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002690352965146303, 'last_eos_top1': 0} +step=97550 lr=6.000000e-04 loss=2.4073 ce=2.4073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6666996089043096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003798368386924267, 'last_eos_top1': 0} +step=97600 lr=6.000000e-04 loss=2.5164 ce=2.5164 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.346402526309248e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002511827740818262, 'last_eos_top1': 0} +step=97650 lr=6.000000e-04 loss=2.7688 ce=2.7688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0204159783315845e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023626829497516155, 'last_eos_top1': 0} +step=97700 lr=6.000000e-04 loss=2.6626 ce=2.6626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.80627851781901e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003793467069044709, 'last_eos_top1': 0} +step=97750 lr=6.000000e-04 loss=1.8972 ce=1.8972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.785351721802726e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031886224169284105, 'last_eos_top1': 0} +step=97800 lr=6.000000e-04 loss=2.6510 ce=2.6510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.608754458487965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002217417350038886, 'last_eos_top1': 0} +step=97850 lr=6.000000e-04 loss=2.4466 ce=2.4466 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.063651537464466e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021202866919338703, 'last_eos_top1': 0} +step=97900 lr=6.000000e-04 loss=2.4670 ce=2.4670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.567678716848604e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002266134135425091, 'last_eos_top1': 0} +step=97950 lr=6.000000e-04 loss=3.3157 ce=3.3157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4564634410780855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002796551212668419, 'last_eos_top1': 0} +step=98000 lr=6.000000e-04 loss=2.8841 ce=2.8841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.317003003961872e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004367213696241379, 'last_eos_top1': 0} +step=98050 lr=6.000000e-04 loss=2.5472 ce=2.5472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.675047188764438e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003866319078952074, 'last_eos_top1': 0} +step=98100 lr=6.000000e-04 loss=2.5973 ce=2.5973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.675742381252348e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004789035767316818, 'last_eos_top1': 0} +step=98150 lr=6.000000e-04 loss=1.9973 ce=1.9973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.243547315738397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00391441909596324, 'last_eos_top1': 0} +step=98200 lr=6.000000e-04 loss=2.8592 ce=2.8592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.880654599517584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036498350091278553, 'last_eos_top1': 0} +step=98250 lr=6.000000e-04 loss=1.8591 ce=1.8591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.377443929115543e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003851155284792185, 'last_eos_top1': 0} +step=98300 lr=6.000000e-04 loss=2.6631 ce=2.6631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.81706717639463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032425904646515846, 'last_eos_top1': 0} +step=98350 lr=6.000000e-04 loss=2.5228 ce=2.5228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0149834654148435e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004566685296595097, 'last_eos_top1': 0} +step=98400 lr=6.000000e-04 loss=2.3648 ce=2.3648 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.731901860097423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004293855279684067, 'last_eos_top1': 0} +step=98450 lr=6.000000e-04 loss=2.9017 ce=2.9017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.443121830874588e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004225796554237604, 'last_eos_top1': 0} +step=98500 lr=6.000000e-04 loss=2.8519 ce=2.8519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.34506489202613e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005671299062669277, 'last_eos_top1': 0} +step=98550 lr=6.000000e-04 loss=2.3346 ce=2.3346 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.203781024145428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003542299848049879, 'last_eos_top1': 0} +step=98600 lr=6.000000e-04 loss=2.6291 ce=2.6291 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.548188710235991e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037323744036257267, 'last_eos_top1': 0} +step=98650 lr=6.000000e-04 loss=2.6644 ce=2.6644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.917793830827577e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003347550518810749, 'last_eos_top1': 0} +step=98700 lr=6.000000e-04 loss=2.6987 ce=2.6987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.852939233998768e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002743212506175041, 'last_eos_top1': 0} +step=98750 lr=6.000000e-04 loss=2.6127 ce=2.6127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.909873834752943e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002558364300057292, 'last_eos_top1': 0} +step=98800 lr=6.000000e-04 loss=2.4111 ce=2.4111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.43727731180843e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038767484948039055, 'last_eos_top1': 0} +step=98850 lr=6.000000e-04 loss=2.5079 ce=2.5079 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.167818355606869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022900723852217197, 'last_eos_top1': 0} +step=98900 lr=6.000000e-04 loss=2.4558 ce=2.4558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.314305665000575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00232813972979784, 'last_eos_top1': 0} +step=98950 lr=6.000000e-04 loss=2.7622 ce=2.7622 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3405814317811746e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029011748265475035, 'last_eos_top1': 0} +step=99000 lr=6.000000e-04 loss=2.4103 ce=2.4103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0031494639406446e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013764193281531334, 'last_eos_top1': 0} +step=99050 lr=6.000000e-04 loss=2.5215 ce=2.5215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1168927964463364e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015089837834239006, 'last_eos_top1': 0} +step=99100 lr=6.000000e-04 loss=3.5156 ce=3.5156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.836395535472548e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002351422095671296, 'last_eos_top1': 0} +step=99150 lr=6.000000e-04 loss=2.3387 ce=2.3387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1827609089086764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030270402785390615, 'last_eos_top1': 0} +step=99200 lr=6.000000e-04 loss=2.5062 ce=2.5062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.74839862868248e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003813972231000662, 'last_eos_top1': 0} +step=99250 lr=6.000000e-04 loss=2.4949 ce=2.4949 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.516409146366641e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003244503168389201, 'last_eos_top1': 0} +step=99300 lr=6.000000e-04 loss=2.2735 ce=2.2735 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8664151159318862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044634408550336957, 'last_eos_top1': 0} +step=99350 lr=6.000000e-04 loss=2.3664 ce=2.3664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6204252208117396e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018726752023212612, 'last_eos_top1': 0} +step=99400 lr=6.000000e-04 loss=2.3931 ce=2.3931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8209730114904232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030575436539947987, 'last_eos_top1': 0} +step=99450 lr=6.000000e-04 loss=2.6905 ce=2.6905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.511320523510221e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007408810197375715, 'last_eos_top1': 0} +[elf_hfds direct rank=6 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=4 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1600000 rows=1591809 buffered=8191 part=openwebtext-t5 +step=99500 lr=6.000000e-04 loss=2.7532 ce=2.7532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.778952643391676e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002663963823579252, 'last_eos_top1': 0} +step=99550 lr=6.000000e-04 loss=3.0623 ce=3.0623 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.4033324633783195e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017867768183350563, 'last_eos_top1': 0} +step=99600 lr=6.000000e-04 loss=2.2583 ce=2.2583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.966433491266798e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007157927611842752, 'last_eos_top1': 0} +step=99650 lr=6.000000e-04 loss=2.8990 ce=2.8990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0298284552118275e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029984640423208475, 'last_eos_top1': 0} +step=99700 lr=6.000000e-04 loss=2.7927 ce=2.7927 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.341933840303682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004734249087050557, 'last_eos_top1': 0} +step=99750 lr=6.000000e-04 loss=3.4331 ce=3.4331 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1008172805304639e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006598387844860554, 'last_eos_top1': 0} +step=99800 lr=6.000000e-04 loss=2.2976 ce=2.2976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.367999958456494e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006402246654033661, 'last_eos_top1': 0} +step=99850 lr=6.000000e-04 loss=2.8329 ce=2.8329 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.215424036781769e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003848822962027043, 'last_eos_top1': 0} +step=99900 lr=6.000000e-04 loss=2.7573 ce=2.7573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.365459623542847e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007818404119461775, 'last_eos_top1': 0} +step=99950 lr=6.000000e-04 loss=2.2128 ce=2.2128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.837928711116547e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016407587099820375, 'last_eos_top1': 0} +step=100000 lr=6.000000e-04 loss=2.5512 ce=2.5512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.419970981776714e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014508070889860392, 'last_eos_top1': 0} +step=100050 lr=6.000000e-04 loss=2.1742 ce=2.1742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.839616015786305e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005408786237239838, 'last_eos_top1': 0} +step=100100 lr=6.000000e-04 loss=2.4904 ce=2.4904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.1992777773411945e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001548656728118658, 'last_eos_top1': 0} +step=100150 lr=6.000000e-04 loss=2.4333 ce=2.4333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7873161343450192e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042936773388646543, 'last_eos_top1': 0} +step=100200 lr=6.000000e-04 loss=2.4511 ce=2.4511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.794489415071439e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000833620666526258, 'last_eos_top1': 0} +step=100250 lr=6.000000e-04 loss=2.4604 ce=2.4604 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4270319601055235e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009555130964145064, 'last_eos_top1': 0} +step=100300 lr=6.000000e-04 loss=2.8119 ce=2.8119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2213045010867063e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011216336861252785, 'last_eos_top1': 0} +step=100350 lr=6.000000e-04 loss=2.5180 ce=2.5180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.291284418897703e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017985461745411158, 'last_eos_top1': 0} +step=100400 lr=6.000000e-04 loss=2.9191 ce=2.9191 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.643996236263774e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012059196596965194, 'last_eos_top1': 0} +step=100450 lr=6.000000e-04 loss=2.7694 ce=2.7694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.106042514555156e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010476566385477781, 'last_eos_top1': 0} +step=100500 lr=6.000000e-04 loss=2.7595 ce=2.7595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.446750037663151e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012106499634683132, 'last_eos_top1': 0} +step=100550 lr=6.000000e-04 loss=1.9712 ce=1.9712 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.754628546128515e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020999284461140633, 'last_eos_top1': 0} +step=100600 lr=6.000000e-04 loss=2.5730 ce=2.5730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3307125300343614e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012314912164583802, 'last_eos_top1': 0} +step=100650 lr=6.000000e-04 loss=3.0249 ce=3.0249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.103898830479011e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013743024319410324, 'last_eos_top1': 0} +step=100700 lr=6.000000e-04 loss=2.3968 ce=2.3968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0666443813533988e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012950410600751638, 'last_eos_top1': 0} +step=100750 lr=6.000000e-04 loss=2.4285 ce=2.4285 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.863135008141398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006348466267809272, 'last_eos_top1': 0} +step=100800 lr=6.000000e-04 loss=2.3683 ce=2.3683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.711839548894204e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007518903585150838, 'last_eos_top1': 0} +step=100850 lr=6.000000e-04 loss=2.3303 ce=2.3303 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.640876679535722e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006963989580981433, 'last_eos_top1': 0} +step=100900 lr=6.000000e-04 loss=2.3956 ce=2.3956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5639502534177154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011673400877043605, 'last_eos_top1': 0} +step=100950 lr=6.000000e-04 loss=3.0679 ce=3.0679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.022207111935131e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001249601598829031, 'last_eos_top1': 0} +step=101000 lr=6.000000e-04 loss=2.3954 ce=2.3954 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3718467825092375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020504207350313663, 'last_eos_top1': 0} +step=101050 lr=6.000000e-04 loss=2.8996 ce=2.8996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1349218766554259e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019762448500841856, 'last_eos_top1': 0} +step=101100 lr=6.000000e-04 loss=2.7289 ce=2.7289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.919591669429792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009379855473525822, 'last_eos_top1': 0} +step=101150 lr=6.000000e-04 loss=2.4015 ce=2.4015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.168576070806012e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007912815781310201, 'last_eos_top1': 0} +step=101200 lr=6.000000e-04 loss=2.2045 ce=2.2045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.094805707514752e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013695858651772141, 'last_eos_top1': 0} +step=101250 lr=6.000000e-04 loss=2.2367 ce=2.2367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.842238457873464e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019563098903745413, 'last_eos_top1': 0} +step=101300 lr=6.000000e-04 loss=2.3009 ce=2.3009 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.517877747886814e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014639702858403325, 'last_eos_top1': 0} +step=101350 lr=6.000000e-04 loss=2.8989 ce=2.8989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2320650613401085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016625470016151667, 'last_eos_top1': 0} +step=101400 lr=6.000000e-04 loss=2.2778 ce=2.2778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.579435881169047e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008350664284080267, 'last_eos_top1': 0} +step=101450 lr=6.000000e-04 loss=2.8093 ce=2.8093 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0932806011114735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001049984130077064, 'last_eos_top1': 0} +step=101500 lr=6.000000e-04 loss=2.2601 ce=2.2601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.808498023427092e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016182315303012729, 'last_eos_top1': 0} +step=101550 lr=6.000000e-04 loss=1.8087 ce=1.8087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6747611526807304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001133539481088519, 'last_eos_top1': 0} +step=101600 lr=6.000000e-04 loss=2.3916 ce=2.3916 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.644035006000195e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018966705538332462, 'last_eos_top1': 0} +step=101650 lr=6.000000e-04 loss=2.8584 ce=2.8584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.039095306827221e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017198343994095922, 'last_eos_top1': 0} +step=101700 lr=6.000000e-04 loss=2.9829 ce=2.9829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.847788533079438e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015938645228743553, 'last_eos_top1': 0} +step=101750 lr=6.000000e-04 loss=2.4104 ce=2.4104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7707367230032105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016659467946738005, 'last_eos_top1': 0} +step=101800 lr=6.000000e-04 loss=3.1334 ce=3.1334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.526966666977387e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028961761854588985, 'last_eos_top1': 0} +step=101850 lr=6.000000e-04 loss=3.0967 ce=3.0967 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.708595825941302e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002564304741099477, 'last_eos_top1': 0} +step=101900 lr=6.000000e-04 loss=2.8512 ce=2.8512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4554949302691966e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028250159230083227, 'last_eos_top1': 0} +step=101950 lr=6.000000e-04 loss=2.1112 ce=2.1112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1109363185823895e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037666992284357548, 'last_eos_top1': 0} +step=102000 lr=6.000000e-04 loss=2.8401 ce=2.8401 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9471082850941457e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003778617363423109, 'last_eos_top1': 0} +step=102050 lr=6.000000e-04 loss=2.7302 ce=2.7302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.216391506721266e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003011434804648161, 'last_eos_top1': 0} +step=102100 lr=6.000000e-04 loss=2.4236 ce=2.4236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3812473197467625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025364700704813004, 'last_eos_top1': 0} +step=102150 lr=6.000000e-04 loss=2.2938 ce=2.2938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.488836904987693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025390100199729204, 'last_eos_top1': 0} +step=102200 lr=6.000000e-04 loss=2.4534 ce=2.4534 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.801956725306809e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023052878677845, 'last_eos_top1': 0} +step=102250 lr=6.000000e-04 loss=3.2951 ce=3.2951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3008839232497849e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002993218367919326, 'last_eos_top1': 0} +step=102300 lr=6.000000e-04 loss=2.5215 ce=2.5215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.14915835892316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027833834756165743, 'last_eos_top1': 0} +step=102350 lr=6.000000e-04 loss=2.8937 ce=2.8937 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.042749191692565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021958183497190475, 'last_eos_top1': 0} +step=102400 lr=6.000000e-04 loss=2.1934 ce=2.1934 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.726014700892847e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002032945863902569, 'last_eos_top1': 0} +step=102450 lr=6.000000e-04 loss=2.5915 ce=2.5915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9330480982898735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015662229852750897, 'last_eos_top1': 0} +step=102500 lr=6.000000e-04 loss=2.6137 ce=2.6137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3113860707962885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001789191970601678, 'last_eos_top1': 0} +step=102550 lr=6.000000e-04 loss=2.6209 ce=2.6209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.870987585396506e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017266166396439075, 'last_eos_top1': 0} +step=102600 lr=6.000000e-04 loss=2.3304 ce=2.3304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.03111720920424e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001299362163990736, 'last_eos_top1': 0} +step=102650 lr=6.000000e-04 loss=2.2286 ce=2.2286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1765785529860295e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015739763621240854, 'last_eos_top1': 0} +step=102700 lr=6.000000e-04 loss=2.5563 ce=2.5563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.498601240105927e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023216442205011845, 'last_eos_top1': 0} +step=102750 lr=6.000000e-04 loss=2.2463 ce=2.2463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.074505341122858e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003273386973887682, 'last_eos_top1': 0} +step=102800 lr=6.000000e-04 loss=2.3207 ce=2.3207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9238962017407175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027994627598673105, 'last_eos_top1': 0} +step=102850 lr=6.000000e-04 loss=1.9571 ce=1.9571 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0915647180809174e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017904555425047874, 'last_eos_top1': 0} +step=102900 lr=6.000000e-04 loss=3.1153 ce=3.1153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.713689905765932e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003478208091109991, 'last_eos_top1': 0} +step=102950 lr=6.000000e-04 loss=2.2763 ce=2.2763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.530249371891841e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017463529948145151, 'last_eos_top1': 0} +step=103000 lr=6.000000e-04 loss=2.0769 ce=2.0769 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.749010364932474e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012176206801086664, 'last_eos_top1': 0} +step=103050 lr=6.000000e-04 loss=2.7677 ce=2.7677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.682137947995216e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002251245779916644, 'last_eos_top1': 0} +step=103100 lr=6.000000e-04 loss=2.6952 ce=2.6952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6595924535067752e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018918835557997227, 'last_eos_top1': 0} +step=103150 lr=6.000000e-04 loss=2.9070 ce=2.9070 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.803936336538754e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017695456044748425, 'last_eos_top1': 0} +step=103200 lr=6.000000e-04 loss=2.2175 ce=2.2175 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1169910067110322e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00242682546377182, 'last_eos_top1': 0} +step=103250 lr=6.000000e-04 loss=2.8175 ce=2.8175 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2729818990919739e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002218372654169798, 'last_eos_top1': 0} +step=103300 lr=6.000000e-04 loss=2.5736 ce=2.5736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.243427949608304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026868716813623905, 'last_eos_top1': 0} +step=103350 lr=6.000000e-04 loss=2.2935 ce=2.2935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.341179763083346e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001239550532773137, 'last_eos_top1': 0} +step=103400 lr=6.000000e-04 loss=2.8374 ce=2.8374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.7726217568852e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027835890650749207, 'last_eos_top1': 0} +step=103450 lr=6.000000e-04 loss=2.5604 ce=2.5604 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.317941147746751e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028827062342315912, 'last_eos_top1': 0} +step=103500 lr=6.000000e-04 loss=2.7791 ce=2.7791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.246417797228787e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032618455588817596, 'last_eos_top1': 0} +step=103550 lr=6.000000e-04 loss=2.4303 ce=2.4303 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8303644537809305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002422632183879614, 'last_eos_top1': 0} +step=103600 lr=6.000000e-04 loss=1.9262 ce=1.9262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5701184464851394e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035614219959825277, 'last_eos_top1': 0} +step=103650 lr=6.000000e-04 loss=2.3559 ce=2.3559 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.780991604202427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003382318653166294, 'last_eos_top1': 0} +step=103700 lr=6.000000e-04 loss=2.3994 ce=2.3994 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.386993042426184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004084347747266293, 'last_eos_top1': 0} +step=103750 lr=6.000000e-04 loss=2.7224 ce=2.7224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0872439816012047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003623402677476406, 'last_eos_top1': 0} +step=103800 lr=6.000000e-04 loss=2.7414 ce=2.7414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1022460967069492e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002026643604040146, 'last_eos_top1': 0} +step=103850 lr=6.000000e-04 loss=2.5897 ce=2.5897 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.291024874371942e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036881067790091038, 'last_eos_top1': 0} +step=103900 lr=6.000000e-04 loss=2.2393 ce=2.2393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.737138730590232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00358627550303936, 'last_eos_top1': 0} +step=103950 lr=6.000000e-04 loss=2.1843 ce=2.1843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.556419637287036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002833761740475893, 'last_eos_top1': 0} +step=104000 lr=6.000000e-04 loss=2.3232 ce=2.3232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.377919923281297e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031103140208870173, 'last_eos_top1': 0} +step=104050 lr=6.000000e-04 loss=1.9795 ce=1.9795 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.350385203608312e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004162526689469814, 'last_eos_top1': 0} +step=104100 lr=6.000000e-04 loss=2.0718 ce=2.0718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.817823875986505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003881375538185239, 'last_eos_top1': 0} +step=104150 lr=6.000000e-04 loss=3.0554 ce=3.0554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2822113098518457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002808027435094118, 'last_eos_top1': 0} +step=104200 lr=6.000000e-04 loss=2.8451 ce=2.8451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.613497134821955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027232617139816284, 'last_eos_top1': 0} +step=104250 lr=6.000000e-04 loss=2.8199 ce=2.8199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2018655221909285e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024526689667254686, 'last_eos_top1': 0} +step=104300 lr=6.000000e-04 loss=2.5982 ce=2.5982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.006477178743808e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013194367056712508, 'last_eos_top1': 0} +step=104350 lr=6.000000e-04 loss=2.1727 ce=2.1727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.485328645387199e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001113795442506671, 'last_eos_top1': 0} +step=104400 lr=6.000000e-04 loss=2.9498 ce=2.9498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.663306289236061e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014198950957506895, 'last_eos_top1': 0} +step=104450 lr=6.000000e-04 loss=2.7170 ce=2.7170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.24134077547933e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001181864645332098, 'last_eos_top1': 0} +step=104500 lr=6.000000e-04 loss=2.5134 ce=2.5134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.131266779149882e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015996122965589166, 'last_eos_top1': 0} +step=104550 lr=6.000000e-04 loss=2.6001 ce=2.6001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.200493771757465e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016170425806194544, 'last_eos_top1': 0} +step=104600 lr=6.000000e-04 loss=2.5992 ce=2.5992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.854860086808912e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002688202541321516, 'last_eos_top1': 0} +step=104650 lr=6.000000e-04 loss=2.8412 ce=2.8412 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.916057827766053e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030193543061614037, 'last_eos_top1': 0} +step=104700 lr=6.000000e-04 loss=2.4356 ce=2.4356 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.968770437903004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00206003338098526, 'last_eos_top1': 0} +step=104750 lr=6.000000e-04 loss=2.9995 ce=2.9995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.953966137894895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011731220874935389, 'last_eos_top1': 0} +step=104800 lr=6.000000e-04 loss=3.1935 ce=3.1935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1071071639889851e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019220798276364803, 'last_eos_top1': 0} +step=104850 lr=6.000000e-04 loss=2.5354 ce=2.5354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2097490980522707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003527599386870861, 'last_eos_top1': 0} +step=104900 lr=6.000000e-04 loss=2.5809 ce=2.5809 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.961678991501685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018757962388917804, 'last_eos_top1': 0} +step=104950 lr=6.000000e-04 loss=2.4109 ce=2.4109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.262154561729403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002491127932444215, 'last_eos_top1': 0} +step=105000 lr=6.000000e-04 loss=2.5022 ce=2.5022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.524600171862403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022587706334888935, 'last_eos_top1': 0} +step=105050 lr=6.000000e-04 loss=2.1167 ce=2.1167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.711320227419492e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024953149259090424, 'last_eos_top1': 0} +step=105100 lr=6.000000e-04 loss=2.0719 ce=2.0719 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.222490810614545e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003957978915423155, 'last_eos_top1': 0} +step=105150 lr=6.000000e-04 loss=2.7448 ce=2.7448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.313688456633827e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002680480480194092, 'last_eos_top1': 0} +step=105200 lr=6.000000e-04 loss=2.2301 ce=2.2301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.293286908956361e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025862790644168854, 'last_eos_top1': 0} +step=105250 lr=6.000000e-04 loss=2.2635 ce=2.2635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.951338152954122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00451439805328846, 'last_eos_top1': 0} +step=105300 lr=6.000000e-04 loss=2.2458 ce=2.2458 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.105448286078172e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031889169476926327, 'last_eos_top1': 0} +step=105350 lr=6.000000e-04 loss=2.4245 ce=2.4245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0429444703040645e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036934700328856707, 'last_eos_top1': 0} +step=105400 lr=6.000000e-04 loss=2.2581 ce=2.2581 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2947146387887187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004973169416189194, 'last_eos_top1': 0} +step=105450 lr=6.000000e-04 loss=3.0182 ce=3.0182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7322927255445393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005857542157173157, 'last_eos_top1': 0} +step=105500 lr=6.000000e-04 loss=2.4781 ce=2.4781 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7039092199411243e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005600950215011835, 'last_eos_top1': 0} +step=105550 lr=6.000000e-04 loss=2.3089 ce=2.3089 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3507362382370047e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0049354322254657745, 'last_eos_top1': 0} +step=105600 lr=6.000000e-04 loss=2.9609 ce=2.9609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2796648661606014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004837770946323872, 'last_eos_top1': 0} +step=105650 lr=6.000000e-04 loss=2.1206 ce=2.1206 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8446152100514155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044531067833304405, 'last_eos_top1': 0} +step=105700 lr=6.000000e-04 loss=2.1394 ce=2.1394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.274238446872914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0050032613798975945, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 + + +[elf_hfds direct rank=2 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 + + +[elf_hfds direct rank=1 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1700000 rows=1691809 buffered=8191 part=openwebtext-t5 +step=105750 lr=6.000000e-04 loss=2.1515 ce=2.1515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.584966973197879e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026761475019156933, 'last_eos_top1': 0} +step=105800 lr=6.000000e-04 loss=2.6989 ce=2.6989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.983549504482653e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017270322423428297, 'last_eos_top1': 0} +step=105850 lr=6.000000e-04 loss=2.7204 ce=2.7204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0926047252723947e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031839367002248764, 'last_eos_top1': 0} +step=105900 lr=6.000000e-04 loss=2.6997 ce=2.6997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.617623744707089e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004227296449244022, 'last_eos_top1': 0} +step=105950 lr=6.000000e-04 loss=2.5573 ce=2.5573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.871868441114202e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004427929874509573, 'last_eos_top1': 0} +step=106000 lr=6.000000e-04 loss=2.7656 ce=2.7656 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0239151126588695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004373439121991396, 'last_eos_top1': 0} +step=106050 lr=6.000000e-04 loss=2.6139 ce=2.6139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1114399967482314e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0045811571180820465, 'last_eos_top1': 0} +step=106100 lr=6.000000e-04 loss=2.2561 ce=2.2561 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1405584700696636e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004795893095433712, 'last_eos_top1': 0} +step=106150 lr=6.000000e-04 loss=2.2421 ce=2.2421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.111354650784051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026521789841353893, 'last_eos_top1': 0} +step=106200 lr=6.000000e-04 loss=2.4140 ce=2.4140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.026052687957417e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032801134511828423, 'last_eos_top1': 0} +step=106250 lr=6.000000e-04 loss=2.5751 ce=2.5751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7897877973591676e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004256825894117355, 'last_eos_top1': 0} +step=106300 lr=6.000000e-04 loss=2.6360 ce=2.6360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.62660635903012e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004370242822915316, 'last_eos_top1': 0} +step=106350 lr=6.000000e-04 loss=2.6766 ce=2.6766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.516774076066213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006502325646579266, 'last_eos_top1': 0} +step=106400 lr=6.000000e-04 loss=2.0621 ce=2.0621 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.729552296746988e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004328911658376455, 'last_eos_top1': 0} +step=106450 lr=6.000000e-04 loss=2.3895 ce=2.3895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.202717147767544e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005953388288617134, 'last_eos_top1': 0} +step=106500 lr=6.000000e-04 loss=2.5017 ce=2.5017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2762654478137847e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005588269792497158, 'last_eos_top1': 0} +step=106550 lr=6.000000e-04 loss=2.3585 ce=2.3585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2617489119293168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005618004128336906, 'last_eos_top1': 0} +step=106600 lr=6.000000e-04 loss=2.4426 ce=2.4426 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.074977242562454e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004937473684549332, 'last_eos_top1': 0} +step=106650 lr=6.000000e-04 loss=2.0857 ce=2.0857 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.269970178138465e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007211466785520315, 'last_eos_top1': 0} +step=106700 lr=6.000000e-04 loss=2.3046 ce=2.3046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4535717026737984e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007079327013343573, 'last_eos_top1': 0} +step=106750 lr=6.000000e-04 loss=2.6785 ce=2.6785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0235682970669586e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005745982751250267, 'last_eos_top1': 0} +step=106800 lr=6.000000e-04 loss=2.1401 ce=2.1401 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.445382390054874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005452524404972792, 'last_eos_top1': 0} +step=106850 lr=6.000000e-04 loss=2.2960 ce=2.2960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1863901313045062e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004555417224764824, 'last_eos_top1': 0} +step=106900 lr=6.000000e-04 loss=2.7517 ce=2.7517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.848070380423451e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005083749070763588, 'last_eos_top1': 0} +step=106950 lr=6.000000e-04 loss=2.8010 ce=2.8010 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0403671694803052e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004092226270586252, 'last_eos_top1': 0} +step=107000 lr=6.000000e-04 loss=2.5248 ce=2.5248 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0972300919529516e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003978559747338295, 'last_eos_top1': 0} +step=107050 lr=6.000000e-04 loss=2.4569 ce=2.4569 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.731489752884954e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004874010570347309, 'last_eos_top1': 0} +step=107100 lr=6.000000e-04 loss=2.4227 ce=2.4227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.572659923316678e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00491143437102437, 'last_eos_top1': 0} +step=107150 lr=6.000000e-04 loss=2.9245 ce=2.9245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2432641243503895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005766422487795353, 'last_eos_top1': 0} +step=107200 lr=6.000000e-04 loss=2.2371 ce=2.2371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.611211584735429e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00591147318482399, 'last_eos_top1': 0} +step=107250 lr=6.000000e-04 loss=2.3550 ce=2.3550 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.980260877753608e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004275043494999409, 'last_eos_top1': 0} +step=107300 lr=6.000000e-04 loss=2.6548 ce=2.6548 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.797973350738175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005134736653417349, 'last_eos_top1': 0} +step=107350 lr=6.000000e-04 loss=2.6851 ce=2.6851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8689150894933846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004010082222521305, 'last_eos_top1': 0} +step=107400 lr=6.000000e-04 loss=2.1225 ce=2.1225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.029557764326455e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005391709040850401, 'last_eos_top1': 0} +step=107450 lr=6.000000e-04 loss=3.0568 ce=3.0568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.452679604582954e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003186342306435108, 'last_eos_top1': 0} +step=107500 lr=6.000000e-04 loss=2.5430 ce=2.5430 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.325195277488092e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006301525980234146, 'last_eos_top1': 0} +step=107550 lr=6.000000e-04 loss=2.1439 ce=2.1439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.355107532523107e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006067036651074886, 'last_eos_top1': 0} +step=107600 lr=6.000000e-04 loss=2.2253 ce=2.2253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5351049518794753e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006504409480839968, 'last_eos_top1': 0} +step=107650 lr=6.000000e-04 loss=2.5726 ce=2.5726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.491652594471816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005872935988008976, 'last_eos_top1': 0} +step=107700 lr=6.000000e-04 loss=2.9663 ce=2.9663 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.977974514185917e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0042585330083966255, 'last_eos_top1': 0} +step=107750 lr=6.000000e-04 loss=2.4514 ce=2.4514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3420751505182125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033410070464015007, 'last_eos_top1': 0} +step=107800 lr=6.000000e-04 loss=2.4028 ce=2.4028 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5481291484320536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033206683583557606, 'last_eos_top1': 0} +step=107850 lr=6.000000e-04 loss=2.2344 ce=2.2344 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.007095180393662e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004020198248326778, 'last_eos_top1': 0} +step=107900 lr=6.000000e-04 loss=2.3586 ce=2.3586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.367529178736731e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004676159005612135, 'last_eos_top1': 0} +step=107950 lr=6.000000e-04 loss=2.1360 ce=2.1360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.041057763970457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004228315316140652, 'last_eos_top1': 0} +step=108000 lr=6.000000e-04 loss=2.4837 ce=2.4837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.857383363647386e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003842470236122608, 'last_eos_top1': 0} +step=108050 lr=6.000000e-04 loss=2.2853 ce=2.2853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0330655641155317e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004795323126018047, 'last_eos_top1': 0} +step=108100 lr=6.000000e-04 loss=2.6081 ce=2.6081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.746647159161512e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029676337726414204, 'last_eos_top1': 0} +step=108150 lr=6.000000e-04 loss=2.8380 ce=2.8380 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.835313459101599e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025237230584025383, 'last_eos_top1': 0} +step=108200 lr=6.000000e-04 loss=2.3837 ce=2.3837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.249757123237941e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011138527188450098, 'last_eos_top1': 0} +step=108250 lr=6.000000e-04 loss=2.7824 ce=2.7824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.105127562885173e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014251258689910173, 'last_eos_top1': 0} +step=108300 lr=6.000000e-04 loss=2.2036 ce=2.2036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.01134184357943e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003380847629159689, 'last_eos_top1': 0} +step=108350 lr=6.000000e-04 loss=2.0718 ce=2.0718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.07725303072948e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043088290840387344, 'last_eos_top1': 0} +step=108400 lr=6.000000e-04 loss=2.3107 ce=2.3107 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.401291441055946e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00497720530256629, 'last_eos_top1': 0} +step=108450 lr=6.000000e-04 loss=2.4857 ce=2.4857 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.261711754021235e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004840406123548746, 'last_eos_top1': 0} +step=108500 lr=6.000000e-04 loss=2.9614 ce=2.9614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.11703547090292e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005026006605476141, 'last_eos_top1': 0} +step=108550 lr=6.000000e-04 loss=2.7522 ce=2.7522 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.422740287031047e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005115800071507692, 'last_eos_top1': 0} +step=108600 lr=6.000000e-04 loss=2.8566 ce=2.8566 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.781343134003691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004816513508558273, 'last_eos_top1': 0} +step=108650 lr=6.000000e-04 loss=1.7716 ce=1.7716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3349984985543415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006679655984044075, 'last_eos_top1': 0} +step=108700 lr=6.000000e-04 loss=2.6198 ce=2.6198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.059835984866368e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005368300713598728, 'last_eos_top1': 0} +step=108750 lr=6.000000e-04 loss=2.3365 ce=2.3365 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.19711068389006e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004281261004507542, 'last_eos_top1': 0} +step=108800 lr=6.000000e-04 loss=2.8230 ce=2.8230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.691895239579026e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034306254237890244, 'last_eos_top1': 0} +step=108850 lr=6.000000e-04 loss=2.9118 ce=2.9118 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.55520852888003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038899530190974474, 'last_eos_top1': 0} +step=108900 lr=6.000000e-04 loss=2.6011 ce=2.6011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.89383888230077e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004507801495492458, 'last_eos_top1': 0} +step=108950 lr=6.000000e-04 loss=2.8779 ce=2.8779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.586870767932851e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004951784387230873, 'last_eos_top1': 0} +step=109000 lr=6.000000e-04 loss=2.6431 ce=2.6431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.796098326347419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004630369134247303, 'last_eos_top1': 0} +step=109050 lr=6.000000e-04 loss=2.2166 ce=2.2166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4781802494544536e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004609611816704273, 'last_eos_top1': 0} +step=109100 lr=6.000000e-04 loss=2.5891 ce=2.5891 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.496771675301716e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004697979427874088, 'last_eos_top1': 0} +step=109150 lr=6.000000e-04 loss=2.3124 ce=2.3124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.906557478534523e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005364995915442705, 'last_eos_top1': 0} +step=109200 lr=6.000000e-04 loss=2.7389 ce=2.7389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6419069147086702e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043829046189785, 'last_eos_top1': 0} +step=109250 lr=6.000000e-04 loss=2.9230 ce=2.9230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2170385161880404e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002980820368975401, 'last_eos_top1': 0} +step=109300 lr=6.000000e-04 loss=2.6863 ce=2.6863 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.924954403875745e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002189244609326124, 'last_eos_top1': 0} +step=109350 lr=6.000000e-04 loss=2.7247 ce=2.7247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.456784831243567e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032370658591389656, 'last_eos_top1': 0} +step=109400 lr=6.000000e-04 loss=3.3241 ce=3.3241 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3459002629097085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00347074749879539, 'last_eos_top1': 0} +step=109450 lr=6.000000e-04 loss=2.9496 ce=2.9496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.249485184322111e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023430311121046543, 'last_eos_top1': 0} +step=109500 lr=6.000000e-04 loss=2.2677 ce=2.2677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5834401640167926e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016489825211465359, 'last_eos_top1': 0} +step=109550 lr=6.000000e-04 loss=2.4626 ce=2.4626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.184662874555215e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015861510764807463, 'last_eos_top1': 0} +step=109600 lr=6.000000e-04 loss=2.5612 ce=2.5612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.531769824709045e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022249321918934584, 'last_eos_top1': 0} +step=109650 lr=6.000000e-04 loss=2.7709 ce=2.7709 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.5187657532515e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012600654736161232, 'last_eos_top1': 0} +step=109700 lr=6.000000e-04 loss=2.4763 ce=2.4763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0002569979405962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006665906403213739, 'last_eos_top1': 0} +step=109750 lr=6.000000e-04 loss=2.8618 ce=2.8618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1168398234294727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043612136505544186, 'last_eos_top1': 0} +step=109800 lr=6.000000e-04 loss=2.6433 ce=2.6433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.327116120199207e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001207845052704215, 'last_eos_top1': 0} +step=109850 lr=6.000000e-04 loss=2.8536 ce=2.8536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2389216357842088e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007922498043626547, 'last_eos_top1': 0} +step=109900 lr=6.000000e-04 loss=2.5006 ce=2.5006 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8341247495263815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000765918055549264, 'last_eos_top1': 0} +step=109950 lr=6.000000e-04 loss=2.7101 ce=2.7101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5428821572859306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008906176080927253, 'last_eos_top1': 0} +step=110000 lr=6.000000e-04 loss=2.6062 ce=2.6062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.046221081604017e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00090833380818367, 'last_eos_top1': 0} +step=110050 lr=6.000000e-04 loss=2.3620 ce=2.3620 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6787178032682277e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006626928225159645, 'last_eos_top1': 0} +step=110100 lr=6.000000e-04 loss=2.6892 ce=2.6892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0159799785469659e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009360401891171932, 'last_eos_top1': 0} +step=110150 lr=6.000000e-04 loss=2.4909 ce=2.4909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8917132769711316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038640113780274987, 'last_eos_top1': 0} +step=110200 lr=6.000000e-04 loss=2.2509 ce=2.2509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.829217116115615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002000453881919384, 'last_eos_top1': 0} +step=110250 lr=6.000000e-04 loss=2.7939 ce=2.7939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.28180817127577e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012531685642898083, 'last_eos_top1': 0} +step=110300 lr=6.000000e-04 loss=2.5304 ce=2.5304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.286523552669678e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011951499618589878, 'last_eos_top1': 0} +step=110350 lr=6.000000e-04 loss=2.8101 ce=2.8101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.033517143398058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012177929747849703, 'last_eos_top1': 0} +step=110400 lr=6.000000e-04 loss=2.1497 ce=2.1497 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.386747034208383e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014223952312022448, 'last_eos_top1': 0} +step=110450 lr=6.000000e-04 loss=2.6488 ce=2.6488 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7304124614602188e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002126815263181925, 'last_eos_top1': 0} +step=110500 lr=6.000000e-04 loss=2.4188 ce=2.4188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6414689980592811e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023336773738265038, 'last_eos_top1': 0} +step=110550 lr=6.000000e-04 loss=2.7634 ce=2.7634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5231524912451277e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027127142529934645, 'last_eos_top1': 0} +step=110600 lr=6.000000e-04 loss=2.4666 ce=2.4666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.4098571783688385e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020102476701140404, 'last_eos_top1': 0} +step=110650 lr=6.000000e-04 loss=2.5220 ce=2.5220 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8499919153546216e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030352147296071053, 'last_eos_top1': 0} +step=110700 lr=6.000000e-04 loss=2.6908 ce=2.6908 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0231210530473618e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002170694526284933, 'last_eos_top1': 0} +step=110750 lr=6.000000e-04 loss=1.8501 ce=1.8501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.602775101782754e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002896683756262064, 'last_eos_top1': 0} +step=110800 lr=6.000000e-04 loss=2.5061 ce=2.5061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.801916813652497e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030875287484377623, 'last_eos_top1': 0} +step=110850 lr=6.000000e-04 loss=2.5865 ce=2.5865 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.665856850915588e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036146268248558044, 'last_eos_top1': 0} +step=110900 lr=6.000000e-04 loss=2.0515 ce=2.0515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.09633628098527e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026348643004894257, 'last_eos_top1': 0} +step=110950 lr=6.000000e-04 loss=2.5527 ce=2.5527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.33911906031426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003133662510663271, 'last_eos_top1': 0} +step=111000 lr=6.000000e-04 loss=2.3010 ce=2.3010 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.988927907514153e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003101018024608493, 'last_eos_top1': 0} +step=111050 lr=6.000000e-04 loss=2.3415 ce=2.3415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0605029274302069e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003032126696780324, 'last_eos_top1': 0} +step=111100 lr=6.000000e-04 loss=2.2517 ce=2.2517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.7567257246992085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032206526957452297, 'last_eos_top1': 0} +step=111150 lr=6.000000e-04 loss=2.6941 ce=2.6941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.218797923589591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027913828380405903, 'last_eos_top1': 0} +step=111200 lr=6.000000e-04 loss=3.2624 ce=3.2624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6750291251810268e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027735214680433273, 'last_eos_top1': 0} +step=111250 lr=6.000000e-04 loss=2.1264 ce=2.1264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9832261386909522e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023493506014347076, 'last_eos_top1': 0} +step=111300 lr=6.000000e-04 loss=2.6550 ce=2.6550 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.192536147864303e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002903045155107975, 'last_eos_top1': 0} +step=111350 lr=6.000000e-04 loss=2.9307 ce=2.9307 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.758959337574197e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001961737871170044, 'last_eos_top1': 0} +step=111400 lr=6.000000e-04 loss=2.8152 ce=2.8152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.676721800933592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011314710136502981, 'last_eos_top1': 0} +step=111450 lr=6.000000e-04 loss=2.6459 ce=2.6459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.416713065642398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008777905022725463, 'last_eos_top1': 0} +step=111500 lr=6.000000e-04 loss=2.6222 ce=2.6222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.861015440837946e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028471971745602787, 'last_eos_top1': 0} +step=111550 lr=6.000000e-04 loss=2.3748 ce=2.3748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.197837708692532e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008503118879161775, 'last_eos_top1': 0} +step=111600 lr=6.000000e-04 loss=2.5549 ce=2.5549 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3913046359448344e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005140427965670824, 'last_eos_top1': 0} +step=111650 lr=6.000000e-04 loss=2.5894 ce=2.5894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.949464942503255e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006139397737570107, 'last_eos_top1': 0} +step=111700 lr=6.000000e-04 loss=2.1084 ce=2.1084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.42712007497903e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005232250550761819, 'last_eos_top1': 0} +step=111750 lr=6.000000e-04 loss=1.9628 ce=1.9628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.519760982977459e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003440755244810134, 'last_eos_top1': 0} +step=111800 lr=6.000000e-04 loss=2.7217 ce=2.7217 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4106138880597427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018248306878376752, 'last_eos_top1': 0} +step=111850 lr=6.000000e-04 loss=2.2781 ce=2.2781 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.491964541055495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008438127115368843, 'last_eos_top1': 0} +step=111900 lr=6.000000e-04 loss=2.7135 ce=2.7135 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.155269940994913e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006984916399233043, 'last_eos_top1': 0} +step=111950 lr=6.000000e-04 loss=3.2455 ce=3.2455 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.092116902640555e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011070799082517624, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1800000 rows=1791809 buffered=8191 part=openwebtext-t5 +step=112000 lr=6.000000e-04 loss=2.8658 ce=2.8658 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.969242243329063e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008246394572779536, 'last_eos_top1': 0} +step=112050 lr=6.000000e-04 loss=2.4204 ce=2.4204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.627833732229192e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007416688604280353, 'last_eos_top1': 0} +step=112100 lr=6.000000e-04 loss=2.9352 ce=2.9352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.198791430098936e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009680530056357384, 'last_eos_top1': 0} +step=112150 lr=6.000000e-04 loss=2.5636 ce=2.5636 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.159639840712771e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007281912257894874, 'last_eos_top1': 0} +step=112200 lr=6.000000e-04 loss=2.1388 ce=2.1388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.581553184834775e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016293921507894993, 'last_eos_top1': 0} +step=112250 lr=6.000000e-04 loss=2.1886 ce=2.1886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2989880764944246e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016672615893185139, 'last_eos_top1': 0} +step=112300 lr=6.000000e-04 loss=2.3859 ce=2.3859 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.468182396522025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025429879315197468, 'last_eos_top1': 0} +step=112350 lr=6.000000e-04 loss=2.3357 ce=2.3357 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.345280780806206e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004229548387229443, 'last_eos_top1': 0} +step=112400 lr=6.000000e-04 loss=3.1227 ce=3.1227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.084027983073611e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029309093952178955, 'last_eos_top1': 0} +step=112450 lr=6.000000e-04 loss=2.7453 ce=2.7453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.844647264690138e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015080403536558151, 'last_eos_top1': 0} +step=112500 lr=6.000000e-04 loss=2.2999 ce=2.2999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5956727515440434e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010969126597046852, 'last_eos_top1': 0} +step=112550 lr=6.000000e-04 loss=2.8840 ce=2.8840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7959644083020976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011377083137631416, 'last_eos_top1': 0} +step=112600 lr=6.000000e-04 loss=2.4216 ce=2.4216 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.447200924711069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010247941827401519, 'last_eos_top1': 0} +step=112650 lr=6.000000e-04 loss=2.9688 ce=2.9688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.958943580073537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.116077515296638e-05, 'last_eos_top1': 0} +step=112700 lr=6.000000e-04 loss=2.7478 ce=2.7478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.757089406368323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00214969040825963, 'last_eos_top1': 0} +step=112750 lr=6.000000e-04 loss=2.5736 ce=2.5736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.879604264511727e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002931297291070223, 'last_eos_top1': 0} +step=112800 lr=6.000000e-04 loss=2.9039 ce=2.9039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.723520083236508e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022934013977646828, 'last_eos_top1': 0} +step=112850 lr=6.000000e-04 loss=2.4763 ce=2.4763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.938164576946292e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001567458501085639, 'last_eos_top1': 0} +step=112900 lr=6.000000e-04 loss=2.3947 ce=2.3947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.081730821781093e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003417673520743847, 'last_eos_top1': 0} +step=112950 lr=6.000000e-04 loss=2.6753 ce=2.6753 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.82510482572252e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004606055095791817, 'last_eos_top1': 0} +step=113000 lr=6.000000e-04 loss=2.2694 ce=2.2694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.813350981043186e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031821243464946747, 'last_eos_top1': 0} +step=113050 lr=6.000000e-04 loss=2.2110 ce=2.2110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.325244041567203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002733873203396797, 'last_eos_top1': 0} +step=113100 lr=6.000000e-04 loss=2.5960 ce=2.5960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.355367415584624e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004115482326596975, 'last_eos_top1': 0} +step=113150 lr=6.000000e-04 loss=2.6527 ce=2.6527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.252136128954589e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035017733462154865, 'last_eos_top1': 0} +step=113200 lr=6.000000e-04 loss=2.1971 ce=2.1971 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.270514066069154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031452528201043606, 'last_eos_top1': 0} +step=113250 lr=6.000000e-04 loss=2.6240 ce=2.6240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.387048764125211e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033819214440882206, 'last_eos_top1': 0} +step=113300 lr=6.000000e-04 loss=2.4743 ce=2.4743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.786182893847581e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003065634984523058, 'last_eos_top1': 0} +step=113350 lr=6.000000e-04 loss=3.0564 ce=3.0564 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8113390979124233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003177214413881302, 'last_eos_top1': 0} +step=113400 lr=6.000000e-04 loss=2.2884 ce=2.2884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6820184656535275e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036209160462021828, 'last_eos_top1': 0} +step=113450 lr=6.000000e-04 loss=2.5354 ce=2.5354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.451642896834528e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00503925234079361, 'last_eos_top1': 0} +step=113500 lr=6.000000e-04 loss=2.6519 ce=2.6519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.01418116982677e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036420479882508516, 'last_eos_top1': 0} +step=113550 lr=6.000000e-04 loss=2.5562 ce=2.5562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.165483788819984e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030686184763908386, 'last_eos_top1': 0} +step=113600 lr=6.000000e-04 loss=2.1739 ce=2.1739 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.938112735748291e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031023118644952774, 'last_eos_top1': 0} +step=113650 lr=6.000000e-04 loss=3.1040 ce=3.1040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.886244823865127e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003853157162666321, 'last_eos_top1': 0} +step=113700 lr=6.000000e-04 loss=2.5623 ce=2.5623 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7244394661684055e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039030732586979866, 'last_eos_top1': 0} +step=113750 lr=6.000000e-04 loss=2.5892 ce=2.5892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7087937723990763e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004770978819578886, 'last_eos_top1': 0} +step=113800 lr=6.000000e-04 loss=2.2942 ce=2.2942 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.639089067699388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004721908830106258, 'last_eos_top1': 0} +step=113850 lr=6.000000e-04 loss=2.6393 ce=2.6393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2656411854550242e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003203107975423336, 'last_eos_top1': 0} +step=113900 lr=6.000000e-04 loss=2.1491 ce=2.1491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.50646916255937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021198480390012264, 'last_eos_top1': 0} +step=113950 lr=6.000000e-04 loss=2.1526 ce=2.1526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6187825571687426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003137882798910141, 'last_eos_top1': 0} +step=114000 lr=6.000000e-04 loss=2.4397 ce=2.4397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.806027613492915e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002270478755235672, 'last_eos_top1': 0} +step=114050 lr=6.000000e-04 loss=2.1655 ce=2.1655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.266121206659591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004474776331335306, 'last_eos_top1': 0} +step=114100 lr=6.000000e-04 loss=2.2688 ce=2.2688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.978336725500412e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004373461473733187, 'last_eos_top1': 0} +step=114150 lr=6.000000e-04 loss=2.3847 ce=2.3847 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.471248757909052e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016854378627613187, 'last_eos_top1': 0} +step=114200 lr=6.000000e-04 loss=2.3835 ce=2.3835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9113556340453215e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015067390631884336, 'last_eos_top1': 0} +step=114250 lr=6.000000e-04 loss=2.3977 ce=2.3977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.823080871574348e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002591160126030445, 'last_eos_top1': 0} +step=114300 lr=6.000000e-04 loss=2.1178 ce=2.1178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.000576831866056e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033702608197927475, 'last_eos_top1': 0} +step=114350 lr=6.000000e-04 loss=2.9035 ce=2.9035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9550413930555806e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004190922249108553, 'last_eos_top1': 0} +step=114400 lr=6.000000e-04 loss=2.0632 ce=2.0632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1799646017607301e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004574860446155071, 'last_eos_top1': 0} +step=114450 lr=6.000000e-04 loss=2.5925 ce=2.5925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.270857971278019e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004012958146631718, 'last_eos_top1': 0} +step=114500 lr=6.000000e-04 loss=2.0381 ce=2.0381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.78183732816251e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004406502936035395, 'last_eos_top1': 0} +step=114550 lr=6.000000e-04 loss=2.2642 ce=2.2642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.626470167248044e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003654139582067728, 'last_eos_top1': 0} +step=114600 lr=6.000000e-04 loss=2.6316 ce=2.6316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.498673196299933e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003752684686332941, 'last_eos_top1': 0} +step=114650 lr=6.000000e-04 loss=2.3362 ce=2.3362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.406592208601069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036312808515504, 'last_eos_top1': 0} +step=114700 lr=6.000000e-04 loss=2.9560 ce=2.9560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.941378054150846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008788518607616425, 'last_eos_top1': 0} +step=114750 lr=6.000000e-04 loss=2.6327 ce=2.6327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0109844879480079e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023201140575110912, 'last_eos_top1': 0} +step=114800 lr=6.000000e-04 loss=2.3023 ce=2.3023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0482516699994449e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031414153054356575, 'last_eos_top1': 0} +step=114850 lr=6.000000e-04 loss=2.3407 ce=2.3407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.98604106521816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003950796090066433, 'last_eos_top1': 0} +step=114900 lr=6.000000e-04 loss=2.4748 ce=2.4748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.995153292635223e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004357224330306053, 'last_eos_top1': 0} +step=114950 lr=6.000000e-04 loss=2.5280 ce=2.5280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4852937460527755e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005346242338418961, 'last_eos_top1': 0} +step=115000 lr=6.000000e-04 loss=2.6169 ce=2.6169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.209236976748798e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0041002631187438965, 'last_eos_top1': 0} +step=115050 lr=6.000000e-04 loss=2.5789 ce=2.5789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.351035830855835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002629270777106285, 'last_eos_top1': 0} +step=115100 lr=6.000000e-04 loss=2.5765 ce=2.5765 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6813300943758804e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016720397397875786, 'last_eos_top1': 0} +step=115150 lr=6.000000e-04 loss=2.6247 ce=2.6247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.55746612715302e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002790252212435007, 'last_eos_top1': 0} +step=115200 lr=6.000000e-04 loss=2.2820 ce=2.2820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6547263486427255e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002180557930842042, 'last_eos_top1': 0} +step=115250 lr=6.000000e-04 loss=2.8649 ce=2.8649 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.808731995988637e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003889520652592182, 'last_eos_top1': 0} +step=115300 lr=6.000000e-04 loss=2.3928 ce=2.3928 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.634533757372992e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002349328715354204, 'last_eos_top1': 0} +step=115350 lr=6.000000e-04 loss=2.2729 ce=2.2729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.659719706978649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003367980942130089, 'last_eos_top1': 0} +step=115400 lr=6.000000e-04 loss=2.4372 ce=2.4372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7593227969191503e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002168309409171343, 'last_eos_top1': 0} +step=115450 lr=6.000000e-04 loss=2.8031 ce=2.8031 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.876936600339832e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024611451663076878, 'last_eos_top1': 0} +step=115500 lr=6.000000e-04 loss=2.7751 ce=2.7751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.630917828762904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017522681737318635, 'last_eos_top1': 0} +step=115550 lr=6.000000e-04 loss=2.5111 ce=2.5111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0191064575337805e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026954617351293564, 'last_eos_top1': 0} +step=115600 lr=6.000000e-04 loss=2.4535 ce=2.4535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.281584657903295e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022770906798541546, 'last_eos_top1': 0} +step=115650 lr=6.000000e-04 loss=2.1363 ce=2.1363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.170900739263743e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017445043195039034, 'last_eos_top1': 0} +step=115700 lr=6.000000e-04 loss=2.4836 ce=2.4836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4768740887520835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026132874190807343, 'last_eos_top1': 0} +step=115750 lr=6.000000e-04 loss=2.1968 ce=2.1968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0357473911426496e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024627347011119127, 'last_eos_top1': 0} +step=115800 lr=6.000000e-04 loss=2.6686 ce=2.6686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.051784344483167e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003392990678548813, 'last_eos_top1': 0} +step=115850 lr=6.000000e-04 loss=2.0661 ce=2.0661 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.445939339348115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002017923863604665, 'last_eos_top1': 0} +step=115900 lr=6.000000e-04 loss=2.5195 ce=2.5195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.543567572705797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004148799926042557, 'last_eos_top1': 0} +step=115950 lr=6.000000e-04 loss=2.6063 ce=2.6063 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.716804925119504e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036739236675202847, 'last_eos_top1': 0} +step=116000 lr=6.000000e-04 loss=2.8947 ce=2.8947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3079094969725702e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030584055930376053, 'last_eos_top1': 0} +step=116050 lr=6.000000e-04 loss=2.2044 ce=2.2044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.386267836613115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030118683353066444, 'last_eos_top1': 0} +step=116100 lr=6.000000e-04 loss=2.7379 ce=2.7379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8649824344029184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038626855239272118, 'last_eos_top1': 0} +step=116150 lr=6.000000e-04 loss=2.2547 ce=2.2547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.559721557801822e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005156937055289745, 'last_eos_top1': 0} +step=116200 lr=6.000000e-04 loss=2.5933 ce=2.5933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.483732017339207e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002592947334051132, 'last_eos_top1': 0} +step=116250 lr=6.000000e-04 loss=2.3451 ce=2.3451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.499391252466012e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032336506992578506, 'last_eos_top1': 0} +step=116300 lr=6.000000e-04 loss=2.3301 ce=2.3301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.932169420295395e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032264881301671267, 'last_eos_top1': 0} +step=116350 lr=6.000000e-04 loss=2.5780 ce=2.5780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1442022696428467e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034413819666951895, 'last_eos_top1': 0} +step=116400 lr=6.000000e-04 loss=2.7524 ce=2.7524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5260635488375556e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032599903643131256, 'last_eos_top1': 0} +step=116450 lr=6.000000e-04 loss=3.1517 ce=3.1517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.74905493017286e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029471435118466616, 'last_eos_top1': 0} +step=116500 lr=6.000000e-04 loss=2.7234 ce=2.7234 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.9412468544906005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032726917415857315, 'last_eos_top1': 0} +step=116550 lr=6.000000e-04 loss=2.6384 ce=2.6384 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.109545898245415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027856952510774136, 'last_eos_top1': 0} +step=116600 lr=6.000000e-04 loss=3.1595 ce=3.1595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.792523552372586e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024721056688576937, 'last_eos_top1': 0} +step=116650 lr=6.000000e-04 loss=2.0891 ce=2.0891 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8218910554423928e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005074865184724331, 'last_eos_top1': 0} +step=116700 lr=6.000000e-04 loss=2.7335 ce=2.7335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.521419214142952e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005902082659304142, 'last_eos_top1': 0} +step=116750 lr=6.000000e-04 loss=2.8976 ce=2.8976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.069649993558414e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003460492705926299, 'last_eos_top1': 0} +step=116800 lr=6.000000e-04 loss=1.9621 ce=1.9621 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.036223693517968e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004513570573180914, 'last_eos_top1': 0} +step=116850 lr=6.000000e-04 loss=2.2925 ce=2.2925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3002037778496742e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037621543742716312, 'last_eos_top1': 0} +step=116900 lr=6.000000e-04 loss=2.0992 ce=2.0992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.38639415531361e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005068231374025345, 'last_eos_top1': 0} +step=116950 lr=6.000000e-04 loss=2.2253 ce=2.2253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5142048798443284e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005779620725661516, 'last_eos_top1': 0} +step=117000 lr=6.000000e-04 loss=2.0234 ce=2.0234 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.651066486374475e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004624322522431612, 'last_eos_top1': 0} +step=117050 lr=6.000000e-04 loss=2.1700 ce=2.1700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0632140806119423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004357478581368923, 'last_eos_top1': 0} +step=117100 lr=6.000000e-04 loss=2.3517 ce=2.3517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3613577013748e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029425909742712975, 'last_eos_top1': 0} +step=117150 lr=6.000000e-04 loss=2.7831 ce=2.7831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8604921428486705e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003019342664629221, 'last_eos_top1': 0} +step=117200 lr=6.000000e-04 loss=2.2085 ce=2.2085 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8014419513056055e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003943099640309811, 'last_eos_top1': 0} +step=117250 lr=6.000000e-04 loss=2.8245 ce=2.8245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.978864697273821e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034045614302158356, 'last_eos_top1': 0} +step=117300 lr=6.000000e-04 loss=3.5657 ce=3.5657 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.281244506884832e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003226070839446038, 'last_eos_top1': 0} +step=117350 lr=6.000000e-04 loss=2.4539 ce=2.4539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3342966970812995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00291749881580472, 'last_eos_top1': 0} +step=117400 lr=6.000000e-04 loss=1.9830 ce=1.9830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.734184585686307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035627332981675863, 'last_eos_top1': 0} +step=117450 lr=6.000000e-04 loss=1.9437 ce=1.9437 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4857380241155624e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002069365233182907, 'last_eos_top1': 0} +step=117500 lr=6.000000e-04 loss=2.7170 ce=2.7170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4842446439142805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023819273337721825, 'last_eos_top1': 0} +step=117550 lr=6.000000e-04 loss=2.0154 ce=2.0154 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.793412128696218e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003292365465313196, 'last_eos_top1': 0} +step=117600 lr=6.000000e-04 loss=2.8104 ce=2.8104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.083046522078803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032779518514871597, 'last_eos_top1': 0} +step=117650 lr=6.000000e-04 loss=2.6076 ce=2.6076 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6629489841288887e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004036339931190014, 'last_eos_top1': 0} +step=117700 lr=6.000000e-04 loss=2.7609 ce=2.7609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.195169822196476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026450580917298794, 'last_eos_top1': 0} +step=117750 lr=6.000000e-04 loss=2.7326 ce=2.7326 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8595150070032105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001188094262033701, 'last_eos_top1': 0} +step=117800 lr=6.000000e-04 loss=2.8475 ce=2.8475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7650606827810407e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004444510559551418, 'last_eos_top1': 0} +step=117850 lr=6.000000e-04 loss=2.1822 ce=2.1822 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.398414037292241e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016638223314657807, 'last_eos_top1': 0} +step=117900 lr=6.000000e-04 loss=2.1273 ce=2.1273 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.612839792476734e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015335471834987402, 'last_eos_top1': 0} +step=117950 lr=6.000000e-04 loss=2.5863 ce=2.5863 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.368757799966261e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014022218529134989, 'last_eos_top1': 0} +step=118000 lr=6.000000e-04 loss=2.1881 ce=2.1881 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0569311750005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005947016179561615, 'last_eos_top1': 0} +step=118050 lr=6.000000e-04 loss=2.0027 ce=2.0027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.06862988913781e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002234244253486395, 'last_eos_top1': 0} +step=118100 lr=6.000000e-04 loss=2.7444 ce=2.7444 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.044402653060388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003767092479392886, 'last_eos_top1': 0} +step=118150 lr=6.000000e-04 loss=2.5097 ce=2.5097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.975310159556102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0053989216685295105, 'last_eos_top1': 0} +step=118200 lr=6.000000e-04 loss=2.7056 ce=2.7056 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.749695108330343e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004344927612692118, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=5 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=2] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=1900000 rows=1891809 buffered=8191 part=openwebtext-t5 +step=118250 lr=6.000000e-04 loss=2.5109 ce=2.5109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.129587862029439e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003929435275495052, 'last_eos_top1': 0} +step=118300 lr=6.000000e-04 loss=2.5977 ce=2.5977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.186263923766091e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028764004819095135, 'last_eos_top1': 0} +step=118350 lr=6.000000e-04 loss=2.9235 ce=2.9235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.159395979921101e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003849748522043228, 'last_eos_top1': 0} +step=118400 lr=6.000000e-04 loss=2.1282 ce=2.1282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.192686901660636e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031480020843446255, 'last_eos_top1': 0} +step=118450 lr=6.000000e-04 loss=2.3593 ce=2.3593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.536876076599583e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019115207251161337, 'last_eos_top1': 0} +step=118500 lr=6.000000e-04 loss=2.7130 ce=2.7130 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2985255933890585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009049263317137957, 'last_eos_top1': 0} +step=118550 lr=6.000000e-04 loss=2.8628 ce=2.8628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7949776015011594e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014166000764817, 'last_eos_top1': 0} +step=118600 lr=6.000000e-04 loss=2.5693 ce=2.5693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.349831553758122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014786531683057547, 'last_eos_top1': 0} +step=118650 lr=6.000000e-04 loss=2.2548 ce=2.2548 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5097706384258345e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017117769457399845, 'last_eos_top1': 0} +step=118700 lr=6.000000e-04 loss=2.5616 ce=2.5616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.841972102236468e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016941959038376808, 'last_eos_top1': 0} +step=118750 lr=6.000000e-04 loss=2.1324 ce=2.1324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.840008846760611e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002538620261475444, 'last_eos_top1': 0} +step=118800 lr=6.000000e-04 loss=2.1910 ce=2.1910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.468557785410667e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036885743029415607, 'last_eos_top1': 0} +step=118850 lr=6.000000e-04 loss=2.9486 ce=2.9486 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.555919076665305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001871146960183978, 'last_eos_top1': 0} +step=118900 lr=6.000000e-04 loss=2.7160 ce=2.7160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.01770284952363e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002506029326468706, 'last_eos_top1': 0} +step=118950 lr=6.000000e-04 loss=2.7800 ce=2.7800 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.120062617905205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003475617151707411, 'last_eos_top1': 0} +step=119000 lr=6.000000e-04 loss=2.4264 ce=2.4264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2180098565295339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002043200423941016, 'last_eos_top1': 0} +step=119050 lr=6.000000e-04 loss=2.5556 ce=2.5556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.675939640539582e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003039110917598009, 'last_eos_top1': 0} +step=119100 lr=6.000000e-04 loss=2.3818 ce=2.3818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.158607569275773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00363266933709383, 'last_eos_top1': 0} +step=119150 lr=6.000000e-04 loss=2.0650 ce=2.0650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.5455654950928874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002164789941161871, 'last_eos_top1': 0} +step=119200 lr=6.000000e-04 loss=2.8714 ce=2.8714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.251393304206431e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017236805288121104, 'last_eos_top1': 0} +step=119250 lr=6.000000e-04 loss=2.5555 ce=2.5555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.630353264976293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002753784880042076, 'last_eos_top1': 0} +step=119300 lr=6.000000e-04 loss=2.4799 ce=2.4799 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5493351384066045e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017361075151711702, 'last_eos_top1': 0} +step=119350 lr=6.000000e-04 loss=2.7282 ce=2.7282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9380241812759778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033450836781412363, 'last_eos_top1': 0} +step=119400 lr=6.000000e-04 loss=2.1872 ce=2.1872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1283689168049023e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004683294333517551, 'last_eos_top1': 0} +step=119450 lr=6.000000e-04 loss=2.6981 ce=2.6981 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.518794867180986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003610472660511732, 'last_eos_top1': 0} +step=119500 lr=6.000000e-04 loss=2.2150 ce=2.2150 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.390767915334436e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004329519346356392, 'last_eos_top1': 0} +step=119550 lr=6.000000e-04 loss=2.8469 ce=2.8469 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.383726708416361e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016612939070910215, 'last_eos_top1': 0} +step=119600 lr=6.000000e-04 loss=2.1424 ce=2.1424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.866306987125427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003183079417794943, 'last_eos_top1': 0} +step=119650 lr=6.000000e-04 loss=2.0961 ce=2.0961 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0298683264409192e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016304722521454096, 'last_eos_top1': 0} +step=119700 lr=6.000000e-04 loss=3.4625 ce=3.4625 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3919850466190837e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021178247407078743, 'last_eos_top1': 0} +step=119750 lr=6.000000e-04 loss=2.1968 ce=2.1968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0372869989369065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028582813683897257, 'last_eos_top1': 0} +step=119800 lr=6.000000e-04 loss=2.7855 ce=2.7855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2755955291795544e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002853130456060171, 'last_eos_top1': 0} +step=119850 lr=6.000000e-04 loss=2.7445 ce=2.7445 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.038863936439157e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016683476278558373, 'last_eos_top1': 0} +step=119900 lr=6.000000e-04 loss=2.7059 ce=2.7059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.090626589459134e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009130436810664833, 'last_eos_top1': 0} +step=119950 lr=6.000000e-04 loss=2.3838 ce=2.3838 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2245301352086244e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001716415397822857, 'last_eos_top1': 0} +step=120000 lr=6.000000e-04 loss=2.2383 ce=2.2383 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.505684389892849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008419376099482179, 'last_eos_top1': 0} +step=120050 lr=6.000000e-04 loss=2.1876 ce=2.1876 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.34957916897838e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001429109601303935, 'last_eos_top1': 0} +step=120100 lr=6.000000e-04 loss=2.4131 ce=2.4131 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6618412195821293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000580164254643023, 'last_eos_top1': 0} +step=120150 lr=6.000000e-04 loss=2.7969 ce=2.7969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6309513739543036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015520857414230704, 'last_eos_top1': 0} +step=120200 lr=6.000000e-04 loss=2.2600 ce=2.2600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.615629339037696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001287813065573573, 'last_eos_top1': 0} +step=120250 lr=6.000000e-04 loss=2.3153 ce=2.3153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.572482794173993e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010644535068422556, 'last_eos_top1': 0} +step=120300 lr=6.000000e-04 loss=2.1910 ce=2.1910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6508860173635185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016065863892436028, 'last_eos_top1': 0} +step=120350 lr=6.000000e-04 loss=2.3780 ce=2.3780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.234955667925533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018735503545030951, 'last_eos_top1': 0} +step=120400 lr=6.000000e-04 loss=3.0601 ce=3.0601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5632293727539945e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002062014304101467, 'last_eos_top1': 0} +step=120450 lr=6.000000e-04 loss=2.5173 ce=2.5173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.34999219578458e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001271803048439324, 'last_eos_top1': 0} +step=120500 lr=6.000000e-04 loss=2.6586 ce=2.6586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.299990334606264e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012992286356166005, 'last_eos_top1': 0} +step=120550 lr=6.000000e-04 loss=2.4738 ce=2.4738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.89366663916735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009068377548828721, 'last_eos_top1': 0} +step=120600 lr=6.000000e-04 loss=2.1857 ce=2.1857 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.698322638636455e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010950437281280756, 'last_eos_top1': 0} +step=120650 lr=6.000000e-04 loss=2.1898 ce=2.1898 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5299710867111571e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012586683733388782, 'last_eos_top1': 0} +step=120700 lr=6.000000e-04 loss=2.6490 ce=2.6490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.04936486625229e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002184343757107854, 'last_eos_top1': 0} +step=120750 lr=6.000000e-04 loss=2.1449 ce=2.1449 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.910078137574601e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002529499353840947, 'last_eos_top1': 0} +step=120800 lr=6.000000e-04 loss=2.7954 ce=2.7954 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1769143384299241e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002876206999644637, 'last_eos_top1': 0} +step=120850 lr=6.000000e-04 loss=2.2203 ce=2.2203 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.212950898363488e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002138456329703331, 'last_eos_top1': 0} +step=120900 lr=6.000000e-04 loss=2.8554 ce=2.8554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.191602213017177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027532284148037434, 'last_eos_top1': 0} +step=120950 lr=6.000000e-04 loss=1.9999 ce=1.9999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2957085548114264e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004425548482686281, 'last_eos_top1': 0} +step=121000 lr=6.000000e-04 loss=2.7627 ce=2.7627 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.390230292803608e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002715981099754572, 'last_eos_top1': 0} +step=121050 lr=6.000000e-04 loss=2.4884 ce=2.4884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5773304059839575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028716842643916607, 'last_eos_top1': 0} +step=121100 lr=6.000000e-04 loss=2.8743 ce=2.8743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.492892008376657e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002468678168952465, 'last_eos_top1': 0} +step=121150 lr=6.000000e-04 loss=2.2899 ce=2.2899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.437577106524259e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016847127117216587, 'last_eos_top1': 0} +step=121200 lr=6.000000e-04 loss=3.0271 ce=3.0271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4666225019464036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020232549868524075, 'last_eos_top1': 0} +step=121250 lr=6.000000e-04 loss=2.6545 ce=2.6545 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1547624530503526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021424544975161552, 'last_eos_top1': 0} +step=121300 lr=6.000000e-04 loss=2.5356 ce=2.5356 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.449170774023514e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004140486940741539, 'last_eos_top1': 0} +step=121350 lr=6.000000e-04 loss=2.3776 ce=2.3776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.453983365237946e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031232223846018314, 'last_eos_top1': 0} +step=121400 lr=6.000000e-04 loss=2.8761 ce=2.8761 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2026686817989685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025308721233159304, 'last_eos_top1': 0} +step=121450 lr=6.000000e-04 loss=2.4532 ce=2.4532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.313965291657951e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001879604533314705, 'last_eos_top1': 0} +step=121500 lr=6.000000e-04 loss=2.8810 ce=2.8810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.836585736076813e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016954541206359863, 'last_eos_top1': 0} +step=121550 lr=6.000000e-04 loss=2.4064 ce=2.4064 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7975002037928789e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003992246463894844, 'last_eos_top1': 0} +step=121600 lr=6.000000e-04 loss=2.4162 ce=2.4162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4499631763319485e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004236976616084576, 'last_eos_top1': 0} +step=121650 lr=6.000000e-04 loss=2.5297 ce=2.5297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.817792614630889e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004146631341427565, 'last_eos_top1': 0} +step=121700 lr=6.000000e-04 loss=2.6177 ce=2.6177 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.269235776315327e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002649466972798109, 'last_eos_top1': 0} +step=121750 lr=6.000000e-04 loss=2.7583 ce=2.7583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3336162965060794e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019854421261698008, 'last_eos_top1': 0} +step=121800 lr=6.000000e-04 loss=2.4698 ce=2.4698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.602442459145095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002631656127050519, 'last_eos_top1': 0} +step=121850 lr=6.000000e-04 loss=2.8214 ce=2.8214 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.669180503289681e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020679144654423, 'last_eos_top1': 0} +step=121900 lr=6.000000e-04 loss=2.4815 ce=2.4815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4512488633044995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026046945713460445, 'last_eos_top1': 0} +step=121950 lr=6.000000e-04 loss=2.5242 ce=2.5242 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9756969272275455e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002958583179861307, 'last_eos_top1': 0} +step=122000 lr=6.000000e-04 loss=2.3047 ce=2.3047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7870833037013654e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017098946264013648, 'last_eos_top1': 0} +step=122050 lr=6.000000e-04 loss=1.8637 ce=1.8637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2728713702235837e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022184811532497406, 'last_eos_top1': 0} +step=122100 lr=6.000000e-04 loss=2.9934 ce=2.9934 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.364009431716113e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011819461360573769, 'last_eos_top1': 0} +step=122150 lr=6.000000e-04 loss=2.8121 ce=2.8121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6427202303457307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001562484772875905, 'last_eos_top1': 0} +step=122200 lr=6.000000e-04 loss=2.8404 ce=2.8404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.600784788839519e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002614885102957487, 'last_eos_top1': 0} +step=122250 lr=6.000000e-04 loss=2.8700 ce=2.8700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.189832791074878e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002753583248704672, 'last_eos_top1': 0} +step=122300 lr=6.000000e-04 loss=2.3536 ce=2.3536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.163560333836358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003402994479984045, 'last_eos_top1': 0} +step=122350 lr=6.000000e-04 loss=2.4018 ce=2.4018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.3534826040267944e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036801048554480076, 'last_eos_top1': 0} +step=122400 lr=6.000000e-04 loss=2.5220 ce=2.5220 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9770228542911354e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016697674291208386, 'last_eos_top1': 0} +step=122450 lr=6.000000e-04 loss=2.1830 ce=2.1830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.595068847062066e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032845681998878717, 'last_eos_top1': 0} +step=122500 lr=6.000000e-04 loss=2.9212 ce=2.9212 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2911779776914045e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000851811608299613, 'last_eos_top1': 0} +step=122550 lr=6.000000e-04 loss=2.6403 ce=2.6403 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5115658647555392e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.663208816666156e-05, 'last_eos_top1': 0} +step=122600 lr=6.000000e-04 loss=2.3159 ce=2.3159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.293845788401086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.383889169432223e-05, 'last_eos_top1': 0} +step=122650 lr=6.000000e-04 loss=2.4207 ce=2.4207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.928893420379609e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.9715086536016315e-05, 'last_eos_top1': 0} +step=122700 lr=6.000000e-04 loss=2.9281 ce=2.9281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.908193437178852e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.104226774070412e-05, 'last_eos_top1': 0} +step=122750 lr=6.000000e-04 loss=2.1363 ce=2.1363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.648357728205156e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.437268843408674e-05, 'last_eos_top1': 0} +step=122800 lr=6.000000e-04 loss=2.4843 ce=2.4843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.536224075010978e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003419871209189296, 'last_eos_top1': 0} +step=122850 lr=6.000000e-04 loss=2.2891 ce=2.2891 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6586799322103616e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.3363409127341583e-05, 'last_eos_top1': 0} +step=122900 lr=6.000000e-04 loss=2.7716 ce=2.7716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0138154468440916e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002210825914517045, 'last_eos_top1': 0} +step=122950 lr=6.000000e-04 loss=2.7413 ce=2.7413 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8800018299079966e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018159463070333004, 'last_eos_top1': 0} +step=123000 lr=6.000000e-04 loss=2.5560 ce=2.5560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.981433453896898e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001060000853613019, 'last_eos_top1': 0} +step=123050 lr=6.000000e-04 loss=2.9302 ce=2.9302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.550782018166501e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005682879127562046, 'last_eos_top1': 0} +step=123100 lr=6.000000e-04 loss=2.6014 ce=2.6014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7398194813722512e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00041379028698429465, 'last_eos_top1': 0} +step=123150 lr=6.000000e-04 loss=2.6996 ce=2.6996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.946656761399936e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003494563279673457, 'last_eos_top1': 0} +step=123200 lr=6.000000e-04 loss=2.5817 ce=2.5817 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5987850424135104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013624265557155013, 'last_eos_top1': 0} +step=123250 lr=6.000000e-04 loss=2.2277 ce=2.2277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.346227342466591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.219156471663155e-05, 'last_eos_top1': 0} +step=123300 lr=6.000000e-04 loss=2.4068 ce=2.4068 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.993347597017419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.123035589349456e-05, 'last_eos_top1': 0} +step=123350 lr=6.000000e-04 loss=2.6098 ce=2.6098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.516738499660278e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.497941260226071e-05, 'last_eos_top1': 0} +step=123400 lr=6.000000e-04 loss=2.7960 ce=2.7960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.633228400052758e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009014687966555357, 'last_eos_top1': 0} +step=123450 lr=6.000000e-04 loss=1.8452 ce=1.8452 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.330270257923985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010042351204901934, 'last_eos_top1': 0} +step=123500 lr=6.000000e-04 loss=2.7108 ce=2.7108 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2623267998133088e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007911754073575139, 'last_eos_top1': 0} +step=123550 lr=6.000000e-04 loss=2.3477 ce=2.3477 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.597509476094274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004984642146155238, 'last_eos_top1': 0} +step=123600 lr=6.000000e-04 loss=2.8101 ce=2.8101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8474005401003524e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009155621519312263, 'last_eos_top1': 0} +step=123650 lr=6.000000e-04 loss=2.5677 ce=2.5677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.687278750585392e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006217792397364974, 'last_eos_top1': 0} +step=123700 lr=6.000000e-04 loss=2.7789 ce=2.7789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.213861524884123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002074812538921833, 'last_eos_top1': 0} +step=123750 lr=6.000000e-04 loss=2.8144 ce=2.8144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.276723418821348e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012921831803396344, 'last_eos_top1': 0} +step=123800 lr=6.000000e-04 loss=2.8280 ce=2.8280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3362542631512042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00167445233091712, 'last_eos_top1': 0} +step=123850 lr=6.000000e-04 loss=1.9969 ce=1.9969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0885704442625865e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001032488769851625, 'last_eos_top1': 0} +step=123900 lr=6.000000e-04 loss=2.1909 ce=2.1909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3908312414278043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012853840598836541, 'last_eos_top1': 0} +step=123950 lr=6.000000e-04 loss=2.5077 ce=2.5077 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.862813173327595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007464827504009008, 'last_eos_top1': 0} +step=124000 lr=6.000000e-04 loss=2.6195 ce=2.6195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2202893786889035e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001979865483008325, 'last_eos_top1': 0} +step=124050 lr=6.000000e-04 loss=1.9750 ce=1.9750 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9419628642936004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001437407045159489, 'last_eos_top1': 0} +step=124100 lr=6.000000e-04 loss=2.7615 ce=2.7615 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.761630280991085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011192276142537594, 'last_eos_top1': 0} +step=124150 lr=6.000000e-04 loss=2.9892 ce=2.9892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.013755981053691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005950917839072645, 'last_eos_top1': 0} +step=124200 lr=6.000000e-04 loss=1.9502 ce=1.9502 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6979637267650105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017166800098493695, 'last_eos_top1': 0} +step=124250 lr=6.000000e-04 loss=2.7155 ce=2.7155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.835912932321662e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014625703915953636, 'last_eos_top1': 0} +step=124300 lr=6.000000e-04 loss=2.4883 ce=2.4883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.655754542502109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010464298538863659, 'last_eos_top1': 0} +step=124350 lr=6.000000e-04 loss=2.5453 ce=2.5453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3772699933033437e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010012437123805285, 'last_eos_top1': 0} +step=124400 lr=6.000000e-04 loss=2.2743 ce=2.2743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.985608368064277e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012191972928121686, 'last_eos_top1': 0} +step=124450 lr=6.000000e-04 loss=2.5882 ce=2.5882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.983935913420282e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012055084807798266, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2000000 rows=1991809 buffered=8191 part=openwebtext-t5 +step=124500 lr=6.000000e-04 loss=3.0584 ce=3.0584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.069727765454445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007863295031711459, 'last_eos_top1': 0} +step=124550 lr=6.000000e-04 loss=2.5320 ce=2.5320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.526735443098005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007660995470359921, 'last_eos_top1': 0} +step=124600 lr=6.000000e-04 loss=2.5270 ce=2.5270 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.220196271693567e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009800129337236285, 'last_eos_top1': 0} +step=124650 lr=6.000000e-04 loss=2.3189 ce=2.3189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.359552804089617e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004664285806939006, 'last_eos_top1': 0} +step=124700 lr=6.000000e-04 loss=2.7310 ce=2.7310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4368808782310225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003063225303776562, 'last_eos_top1': 0} +step=124750 lr=6.000000e-04 loss=2.2467 ce=2.2467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5339377316413447e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020633613166864961, 'last_eos_top1': 0} +step=124800 lr=6.000000e-04 loss=2.5144 ce=2.5144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0389893759565894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002445051504764706, 'last_eos_top1': 0} +step=124850 lr=6.000000e-04 loss=2.2390 ce=2.2390 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.117923476267606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003334433422423899, 'last_eos_top1': 0} +step=124900 lr=6.000000e-04 loss=1.9703 ce=1.9703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.325987563584931e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009140886832028627, 'last_eos_top1': 0} +step=124950 lr=6.000000e-04 loss=2.5943 ce=2.5943 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.198009148443816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008580666035413742, 'last_eos_top1': 0} +step=125000 lr=6.000000e-04 loss=2.7932 ce=2.7932 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.581715529639041e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011856702622026205, 'last_eos_top1': 0} +step=125050 lr=6.000000e-04 loss=2.4227 ce=2.4227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2311774046102073e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008714250288903713, 'last_eos_top1': 0} +step=125100 lr=6.000000e-04 loss=2.6392 ce=2.6392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8016278772847727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007742008892819285, 'last_eos_top1': 0} +step=125150 lr=6.000000e-04 loss=2.3789 ce=2.3789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.400149919092655e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015867498004809022, 'last_eos_top1': 0} +step=125200 lr=6.000000e-04 loss=3.1036 ce=3.1036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6914398151566274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012660010252147913, 'last_eos_top1': 0} +step=125250 lr=6.000000e-04 loss=2.9829 ce=2.9829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0848615349677857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009942204924300313, 'last_eos_top1': 0} +step=125300 lr=6.000000e-04 loss=2.4950 ce=2.4950 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.246869477559812e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011104235891252756, 'last_eos_top1': 0} +step=125350 lr=6.000000e-04 loss=2.9973 ce=2.9973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.515693374036346e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013508485862985253, 'last_eos_top1': 0} +step=125400 lr=6.000000e-04 loss=2.4589 ce=2.4589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.414046533289365e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001303879776969552, 'last_eos_top1': 0} +step=125450 lr=6.000000e-04 loss=2.7501 ce=2.7501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0639704416680615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019899490289390087, 'last_eos_top1': 0} +step=125500 lr=6.000000e-04 loss=2.4963 ce=2.4963 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.737806532124523e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008146236650645733, 'last_eos_top1': 0} +step=125550 lr=6.000000e-04 loss=2.4764 ce=2.4764 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5171455035888357e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013632981572300196, 'last_eos_top1': 0} +step=125600 lr=6.000000e-04 loss=2.5162 ce=2.5162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.394706022343598e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001300342846661806, 'last_eos_top1': 0} +step=125650 lr=6.000000e-04 loss=2.1782 ce=2.1782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.273650352435652e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027092639356851578, 'last_eos_top1': 0} +step=125700 lr=6.000000e-04 loss=3.0316 ce=3.0316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.247865268145688e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027263453230261803, 'last_eos_top1': 0} +step=125750 lr=6.000000e-04 loss=2.2801 ce=2.2801 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6334861356881447e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020367619581520557, 'last_eos_top1': 0} +step=125800 lr=6.000000e-04 loss=2.3690 ce=2.3690 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.582349775882903e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.002522283000871539, 'last_eos_top1': 0} +step=125850 lr=6.000000e-04 loss=2.6671 ce=2.6671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1472585558512947e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030906700994819403, 'last_eos_top1': 0} +step=125900 lr=6.000000e-04 loss=2.8126 ce=2.8126 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.262630687255296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016212466871365905, 'last_eos_top1': 0} +step=125950 lr=6.000000e-04 loss=2.5917 ce=2.5917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.262374093537801e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018071269150823355, 'last_eos_top1': 0} +step=126000 lr=6.000000e-04 loss=2.9222 ce=2.9222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.959554083441617e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006483182078227401, 'last_eos_top1': 0} +step=126050 lr=6.000000e-04 loss=2.3370 ce=2.3370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.447862243570853e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000571352313272655, 'last_eos_top1': 0} +step=126100 lr=6.000000e-04 loss=2.5358 ce=2.5358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7073392655220232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004117222561035305, 'last_eos_top1': 0} +step=126150 lr=6.000000e-04 loss=2.7140 ce=2.7140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.716486505960347e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046595733147114515, 'last_eos_top1': 0} +step=126200 lr=6.000000e-04 loss=2.4707 ce=2.4707 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4813333513739053e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006625325768254697, 'last_eos_top1': 0} +step=126250 lr=6.000000e-04 loss=2.5416 ce=2.5416 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1964880236046156e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005156407132744789, 'last_eos_top1': 0} +step=126300 lr=6.000000e-04 loss=2.2434 ce=2.2434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1747005121142138e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008631506934762001, 'last_eos_top1': 0} +step=126350 lr=6.000000e-04 loss=2.3543 ce=2.3543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.437169081938919e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019352255621924996, 'last_eos_top1': 0} +step=126400 lr=6.000000e-04 loss=2.5774 ce=2.5774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.672718205256388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018210624111816287, 'last_eos_top1': 0} +step=126450 lr=6.000000e-04 loss=2.2373 ce=2.2373 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3711157862417167e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012863286538049579, 'last_eos_top1': 0} +step=126500 lr=6.000000e-04 loss=2.6717 ce=2.6717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1925097826169804e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002326907590031624, 'last_eos_top1': 0} +step=126550 lr=6.000000e-04 loss=2.2462 ce=2.2462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1644544883893104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00199934933334589, 'last_eos_top1': 0} +step=126600 lr=6.000000e-04 loss=2.0835 ce=2.0835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5544386517140083e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002209650818258524, 'last_eos_top1': 0} +step=126650 lr=6.000000e-04 loss=2.4331 ce=2.4331 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7734359971655067e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002135674934834242, 'last_eos_top1': 0} +step=126700 lr=6.000000e-04 loss=2.5121 ce=2.5121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.784349130204646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023775696754455566, 'last_eos_top1': 0} +step=126750 lr=6.000000e-04 loss=2.2805 ce=2.2805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8144194149936084e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015087350038811564, 'last_eos_top1': 0} +step=126800 lr=6.000000e-04 loss=2.5853 ce=2.5853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6301130472129444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002229051198810339, 'last_eos_top1': 0} +step=126850 lr=6.000000e-04 loss=2.9814 ce=2.9814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5526100014540134e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018123333575204015, 'last_eos_top1': 0} +step=126900 lr=6.000000e-04 loss=2.3110 ce=2.3110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1973471525125206e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003041204996407032, 'last_eos_top1': 0} +step=126950 lr=6.000000e-04 loss=2.0434 ce=2.0434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.45759724243544e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002724663121625781, 'last_eos_top1': 0} +step=127000 lr=6.000000e-04 loss=2.6323 ce=2.6323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5635048334370367e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001630566781386733, 'last_eos_top1': 0} +step=127050 lr=6.000000e-04 loss=2.5647 ce=2.5647 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.776971086859703e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022513242438435555, 'last_eos_top1': 0} +step=127100 lr=6.000000e-04 loss=2.9760 ce=2.9760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8958829716430046e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022218108642846346, 'last_eos_top1': 0} +step=127150 lr=6.000000e-04 loss=2.4193 ce=2.4193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.230981514614541e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003072600346058607, 'last_eos_top1': 0} +step=127200 lr=6.000000e-04 loss=2.2138 ce=2.2138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8728725232649595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003941888455301523, 'last_eos_top1': 0} +step=127250 lr=6.000000e-04 loss=2.2511 ce=2.2511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1070348894863855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037779919803142548, 'last_eos_top1': 0} +step=127300 lr=6.000000e-04 loss=2.1910 ce=2.1910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.751918590452988e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033781344536691904, 'last_eos_top1': 0} +step=127350 lr=6.000000e-04 loss=2.5136 ce=2.5136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.18662011725246e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002594791352748871, 'last_eos_top1': 0} +step=127400 lr=6.000000e-04 loss=3.1148 ce=3.1148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7109551865578396e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019498297479003668, 'last_eos_top1': 0} +step=127450 lr=6.000000e-04 loss=2.4772 ce=2.4772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.632313903130125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002051474992185831, 'last_eos_top1': 0} +step=127500 lr=6.000000e-04 loss=2.4171 ce=2.4171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1895402824156918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031139091588556767, 'last_eos_top1': 0} +step=127550 lr=6.000000e-04 loss=2.1546 ce=2.1546 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.755466423172038e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031469259411096573, 'last_eos_top1': 0} +step=127600 lr=6.000000e-04 loss=2.4130 ce=2.4130 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.977550138894003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001469169044867158, 'last_eos_top1': 0} +step=127650 lr=6.000000e-04 loss=2.8386 ce=2.8386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5946491203067126e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002471711952239275, 'last_eos_top1': 0} +step=127700 lr=6.000000e-04 loss=2.4531 ce=2.4531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8041957901004935e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020078145898878574, 'last_eos_top1': 0} +step=127750 lr=6.000000e-04 loss=2.3287 ce=2.3287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.744173571045394e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038038885686546564, 'last_eos_top1': 0} +step=127800 lr=6.000000e-04 loss=2.6886 ce=2.6886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.253011295280885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002685592044144869, 'last_eos_top1': 0} +step=127850 lr=6.000000e-04 loss=2.8903 ce=2.8903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.225510565447621e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017592899966984987, 'last_eos_top1': 0} +step=127900 lr=6.000000e-04 loss=2.2109 ce=2.2109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.155827016598778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033404696732759476, 'last_eos_top1': 0} +step=127950 lr=6.000000e-04 loss=2.4629 ce=2.4629 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.18338822782971e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033341876696795225, 'last_eos_top1': 0} +step=128000 lr=6.000000e-04 loss=3.3307 ce=3.3307 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.672497198043857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032817574683576822, 'last_eos_top1': 0} +step=128050 lr=6.000000e-04 loss=2.6751 ce=2.6751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0278544070606586e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034337914548814297, 'last_eos_top1': 0} +step=128100 lr=6.000000e-04 loss=2.3874 ce=2.3874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6500691749097314e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030059246346354485, 'last_eos_top1': 0} +step=128150 lr=6.000000e-04 loss=2.3906 ce=2.3906 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.430113110880484e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004646162036806345, 'last_eos_top1': 0} +step=128200 lr=6.000000e-04 loss=2.7379 ce=2.7379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0181586225808132e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00403817743062973, 'last_eos_top1': 0} +step=128250 lr=6.000000e-04 loss=2.4289 ce=2.4289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.940279384551104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0045815324410796165, 'last_eos_top1': 0} +step=128300 lr=6.000000e-04 loss=2.7297 ce=2.7297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.73036925541237e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034578493796288967, 'last_eos_top1': 0} +step=128350 lr=6.000000e-04 loss=2.1386 ce=2.1386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.673193183814874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031542559154331684, 'last_eos_top1': 0} +step=128400 lr=6.000000e-04 loss=2.3668 ce=2.3668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.962436368165072e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027496693655848503, 'last_eos_top1': 0} +step=128450 lr=6.000000e-04 loss=2.6491 ce=2.6491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1631924280663952e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003997678868472576, 'last_eos_top1': 0} +step=128500 lr=6.000000e-04 loss=2.6074 ce=2.6074 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.020313882501796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019461691845208406, 'last_eos_top1': 0} +step=128550 lr=6.000000e-04 loss=2.9090 ce=2.9090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.112025509239174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038994774222373962, 'last_eos_top1': 0} +step=128600 lr=6.000000e-04 loss=2.5921 ce=2.5921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.383031748351641e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014683633344247937, 'last_eos_top1': 0} +step=128650 lr=6.000000e-04 loss=2.7184 ce=2.7184 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.225259080063552e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031903726048767567, 'last_eos_top1': 0} +step=128700 lr=6.000000e-04 loss=2.1519 ce=2.1519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.453356945712585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031944303773343563, 'last_eos_top1': 0} +step=128750 lr=6.000000e-04 loss=2.0584 ce=2.0584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.227914233633783e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002203715033829212, 'last_eos_top1': 0} +step=128800 lr=6.000000e-04 loss=2.5052 ce=2.5052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.184948117355816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030563599430024624, 'last_eos_top1': 0} +step=128850 lr=6.000000e-04 loss=2.5838 ce=2.5838 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1634172551566735e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028350409120321274, 'last_eos_top1': 0} +step=128900 lr=6.000000e-04 loss=2.4021 ce=2.4021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1730112419172656e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029826918616890907, 'last_eos_top1': 0} +step=128950 lr=6.000000e-04 loss=2.4133 ce=2.4133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.945256584207527e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021379359532147646, 'last_eos_top1': 0} +step=129000 lr=6.000000e-04 loss=2.6755 ce=2.6755 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.735840310691856e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018234197050333023, 'last_eos_top1': 0} +step=129050 lr=6.000000e-04 loss=2.3836 ce=2.3836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.255149284901563e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026420108042657375, 'last_eos_top1': 0} +step=129100 lr=6.000000e-04 loss=2.0676 ce=2.0676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.680245289520826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002087746746838093, 'last_eos_top1': 0} +step=129150 lr=6.000000e-04 loss=2.5861 ce=2.5861 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4055922292754985e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021298800129443407, 'last_eos_top1': 0} +step=129200 lr=6.000000e-04 loss=2.4663 ce=2.4663 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0059307896881364e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016802626196295023, 'last_eos_top1': 0} +step=129250 lr=6.000000e-04 loss=2.1720 ce=2.1720 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1248699593124911e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00214221328496933, 'last_eos_top1': 0} +step=129300 lr=6.000000e-04 loss=2.2818 ce=2.2818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.051508206728613e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002784226555377245, 'last_eos_top1': 0} +step=129350 lr=6.000000e-04 loss=2.6286 ce=2.6286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7272770492127165e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014303710777312517, 'last_eos_top1': 0} +step=129400 lr=6.000000e-04 loss=2.7655 ce=2.7655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.70287533669034e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018001915886998177, 'last_eos_top1': 0} +step=129450 lr=6.000000e-04 loss=2.6944 ce=2.6944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4809097592660692e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022258497774600983, 'last_eos_top1': 0} +step=129500 lr=6.000000e-04 loss=2.4718 ce=2.4718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.10723487220821e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005745806265622377, 'last_eos_top1': 0} +step=129550 lr=6.000000e-04 loss=2.5986 ce=2.5986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.250648534158245e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009327771840617061, 'last_eos_top1': 0} +step=129600 lr=6.000000e-04 loss=2.7478 ce=2.7478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0048812328022905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012077676365152001, 'last_eos_top1': 0} +step=129650 lr=6.000000e-04 loss=2.5699 ce=2.5699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.009392670122907e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010990654118359089, 'last_eos_top1': 0} +step=129700 lr=6.000000e-04 loss=2.3960 ce=2.3960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0540389666857664e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011581615544855595, 'last_eos_top1': 0} +step=129750 lr=6.000000e-04 loss=2.5456 ce=2.5456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.618605998985004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013868361711502075, 'last_eos_top1': 0} +step=129800 lr=6.000000e-04 loss=2.8105 ce=2.8105 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.185008037893567e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018988663796335459, 'last_eos_top1': 0} +step=129850 lr=6.000000e-04 loss=2.9374 ce=2.9374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.75407193537103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016323586460202932, 'last_eos_top1': 0} +step=129900 lr=6.000000e-04 loss=1.9102 ce=1.9102 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6778602002887055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015423833392560482, 'last_eos_top1': 0} +step=129950 lr=6.000000e-04 loss=2.5682 ce=2.5682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.958824997127522e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000935432908590883, 'last_eos_top1': 0} +step=130000 lr=6.000000e-04 loss=2.3919 ce=2.3919 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.650482752476819e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013862098567187786, 'last_eos_top1': 0} +step=130050 lr=6.000000e-04 loss=3.2556 ce=3.2556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.063242603384424e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008385317050851882, 'last_eos_top1': 0} +step=130100 lr=6.000000e-04 loss=2.2869 ce=2.2869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.536292625649367e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001278783194720745, 'last_eos_top1': 0} +step=130150 lr=6.000000e-04 loss=2.4825 ce=2.4825 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5786705009522848e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017809122800827026, 'last_eos_top1': 0} +step=130200 lr=6.000000e-04 loss=2.5721 ce=2.5721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.377908226160798e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001197671052068472, 'last_eos_top1': 0} +step=130250 lr=6.000000e-04 loss=2.2555 ce=2.2555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3775705156149343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012064091861248016, 'last_eos_top1': 0} +step=130300 lr=6.000000e-04 loss=2.8134 ce=2.8134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0457739335834049e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016990129370242357, 'last_eos_top1': 0} +step=130350 lr=6.000000e-04 loss=2.0544 ce=2.0544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.536908247449901e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024163187481462955, 'last_eos_top1': 0} +step=130400 lr=6.000000e-04 loss=2.4159 ce=2.4159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.294451511872467e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016631113830953836, 'last_eos_top1': 0} +step=130450 lr=6.000000e-04 loss=3.0231 ce=3.0231 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.4294472394976765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002621493535116315, 'last_eos_top1': 0} +step=130500 lr=6.000000e-04 loss=2.3026 ce=2.3026 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.207788151688874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004477285780012608, 'last_eos_top1': 0} +step=130550 lr=6.000000e-04 loss=2.9314 ce=2.9314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.838157226709882e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003512911032885313, 'last_eos_top1': 0} +step=130600 lr=6.000000e-04 loss=2.5965 ce=2.5965 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.985829830024159e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030598121229559183, 'last_eos_top1': 0} +step=130650 lr=6.000000e-04 loss=2.6104 ce=2.6104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.956260909239063e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00437746150419116, 'last_eos_top1': 0} +step=130700 lr=6.000000e-04 loss=2.5311 ce=2.5311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.937176421051845e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002640076447278261, 'last_eos_top1': 0} +[elf_hfds direct rank=3 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 + + +[elf_hfds direct rank=2 worker=3] docs=2100000 rows=2091809 buffered=8191 part=openwebtext-t5 +step=130750 lr=6.000000e-04 loss=2.8368 ce=2.8368 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.080706953071058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023678848519921303, 'last_eos_top1': 0} +step=130800 lr=6.000000e-04 loss=2.1530 ce=2.1530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.912802640115842e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001995829399675131, 'last_eos_top1': 0} +step=130850 lr=6.000000e-04 loss=2.5445 ce=2.5445 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.076070349081419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011519441613927484, 'last_eos_top1': 0} +step=130900 lr=6.000000e-04 loss=2.7327 ce=2.7327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7603433661861345e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016798243159428239, 'last_eos_top1': 0} +step=130950 lr=6.000000e-04 loss=2.6319 ce=2.6319 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.654941898887046e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001889328472316265, 'last_eos_top1': 0} +step=131000 lr=6.000000e-04 loss=2.4611 ce=2.4611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.606818376691081e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00268815690651536, 'last_eos_top1': 0} +step=131050 lr=6.000000e-04 loss=2.8376 ce=2.8376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.470585281524109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028526796959340572, 'last_eos_top1': 0} +step=131100 lr=6.000000e-04 loss=2.5371 ce=2.5371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.329114280059002e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027313148602843285, 'last_eos_top1': 0} +step=131150 lr=6.000000e-04 loss=2.9671 ce=2.9671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.225598554010503e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001831857138313353, 'last_eos_top1': 0} +step=131200 lr=6.000000e-04 loss=2.1757 ce=2.1757 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.174831923999591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013370767701417208, 'last_eos_top1': 0} +step=131250 lr=6.000000e-04 loss=2.1763 ce=2.1763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.556679752771743e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002875334583222866, 'last_eos_top1': 0} +step=131300 lr=6.000000e-04 loss=2.3780 ce=2.3780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.979897418408655e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002100258832797408, 'last_eos_top1': 0} +step=131350 lr=6.000000e-04 loss=2.6176 ce=2.6176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.866538125090301e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003020789474248886, 'last_eos_top1': 0} +step=131400 lr=6.000000e-04 loss=2.0544 ce=2.0544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.604675142327324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032913386821746826, 'last_eos_top1': 0} +step=131450 lr=6.000000e-04 loss=2.6794 ce=2.6794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6833598642260768e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024774037301540375, 'last_eos_top1': 0} +step=131500 lr=6.000000e-04 loss=2.7134 ce=2.7134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.728889275109395e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023495545610785484, 'last_eos_top1': 0} +step=131550 lr=6.000000e-04 loss=2.5039 ce=2.5039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.17458807330695e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014929566532373428, 'last_eos_top1': 0} +step=131600 lr=6.000000e-04 loss=2.7191 ce=2.7191 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.33328147866996e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015881452709436417, 'last_eos_top1': 0} +step=131650 lr=6.000000e-04 loss=2.0113 ce=2.0113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.273935468925629e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018213592702522874, 'last_eos_top1': 0} +step=131700 lr=6.000000e-04 loss=2.2820 ce=2.2820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.654969077615533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018584206700325012, 'last_eos_top1': 0} +step=131750 lr=6.000000e-04 loss=2.4992 ce=2.4992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.061269919475308e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016624724958091974, 'last_eos_top1': 0} +step=131800 lr=6.000000e-04 loss=2.5064 ce=2.5064 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.389215973787941e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00098436139523983, 'last_eos_top1': 0} +step=131850 lr=6.000000e-04 loss=2.1249 ce=2.1249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.589817763189785e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021999329328536987, 'last_eos_top1': 0} +step=131900 lr=6.000000e-04 loss=2.5487 ce=2.5487 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0269569581141695e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002369216177612543, 'last_eos_top1': 0} +step=131950 lr=6.000000e-04 loss=2.7459 ce=2.7459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0067960829474032e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020763587672263384, 'last_eos_top1': 0} +step=132000 lr=6.000000e-04 loss=2.4337 ce=2.4337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8929432523436844e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001808584202080965, 'last_eos_top1': 0} +step=132050 lr=6.000000e-04 loss=2.9305 ce=2.9305 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3148953662021086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014259670861065388, 'last_eos_top1': 0} +step=132100 lr=6.000000e-04 loss=2.4391 ce=2.4391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7093585711336345e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006984960054978728, 'last_eos_top1': 0} +step=132150 lr=6.000000e-04 loss=2.7799 ce=2.7799 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.173371512981248e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025459621101617813, 'last_eos_top1': 0} +step=132200 lr=6.000000e-04 loss=2.2267 ce=2.2267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.244037503056461e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033837961964309216, 'last_eos_top1': 0} +step=132250 lr=6.000000e-04 loss=2.5030 ce=2.5030 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.499241524375975e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032092290930449963, 'last_eos_top1': 0} +step=132300 lr=6.000000e-04 loss=2.7265 ce=2.7265 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.292500645737164e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002268110401928425, 'last_eos_top1': 0} +step=132350 lr=6.000000e-04 loss=2.7138 ce=2.7138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.082337909494527e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032899370416998863, 'last_eos_top1': 0} +step=132400 lr=6.000000e-04 loss=2.4125 ce=2.4125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6156681087741163e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018854399677366018, 'last_eos_top1': 0} +step=132450 lr=6.000000e-04 loss=2.3868 ce=2.3868 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7198124169226503e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022088794503360987, 'last_eos_top1': 0} +step=132500 lr=6.000000e-04 loss=2.3144 ce=2.3144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.310983058530837e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000361207639798522, 'last_eos_top1': 0} +step=132550 lr=6.000000e-04 loss=2.9029 ce=2.9029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1029310321173398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018166691006626934, 'last_eos_top1': 0} +step=132600 lr=6.000000e-04 loss=2.5375 ce=2.5375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.618915232233121e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004899721243418753, 'last_eos_top1': 0} +step=132650 lr=6.000000e-04 loss=2.7026 ce=2.7026 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3444615635526134e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014403476379811764, 'last_eos_top1': 0} +step=132700 lr=6.000000e-04 loss=2.8557 ce=2.8557 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.87282398151001e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015509603545069695, 'last_eos_top1': 0} +step=132750 lr=6.000000e-04 loss=2.9948 ce=2.9948 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.380200491345022e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023813736625015736, 'last_eos_top1': 0} +step=132800 lr=6.000000e-04 loss=2.0957 ce=2.0957 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.167465365550015e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002942043822258711, 'last_eos_top1': 0} +step=132850 lr=6.000000e-04 loss=2.2495 ce=2.2495 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.41488578467397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034592407755553722, 'last_eos_top1': 0} +step=132900 lr=6.000000e-04 loss=1.8500 ce=1.8500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.93758101988351e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021620222833007574, 'last_eos_top1': 0} +step=132950 lr=6.000000e-04 loss=2.6511 ce=2.6511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6695503189985175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015729519072920084, 'last_eos_top1': 0} +step=133000 lr=6.000000e-04 loss=2.1263 ce=2.1263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.861483608285198e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020133797079324722, 'last_eos_top1': 0} +step=133050 lr=6.000000e-04 loss=2.8099 ce=2.8099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.415759576659184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006880335276946425, 'last_eos_top1': 0} +step=133100 lr=6.000000e-04 loss=2.3853 ce=2.3853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0321096851839684e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000849739764817059, 'last_eos_top1': 0} +step=133150 lr=6.000000e-04 loss=1.8932 ce=1.8932 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1900090612471104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006235271575860679, 'last_eos_top1': 0} +step=133200 lr=6.000000e-04 loss=2.3883 ce=2.3883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.912317192269256e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008164475439116359, 'last_eos_top1': 0} +step=133250 lr=6.000000e-04 loss=2.4185 ce=2.4185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5123622435785364e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013258133549243212, 'last_eos_top1': 0} +step=133300 lr=6.000000e-04 loss=2.1909 ce=2.1909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.110093297844287e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028178709908388555, 'last_eos_top1': 0} +step=133350 lr=6.000000e-04 loss=2.7175 ce=2.7175 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7619421441850136e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003205646644346416, 'last_eos_top1': 0} +step=133400 lr=6.000000e-04 loss=2.6142 ce=2.6142 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3385199458280113e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011568366317078471, 'last_eos_top1': 0} +step=133450 lr=6.000000e-04 loss=2.4538 ce=2.4538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.065670054842485e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017624759348109365, 'last_eos_top1': 0} +step=133500 lr=6.000000e-04 loss=3.0961 ce=3.0961 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.584850325452862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013373023830354214, 'last_eos_top1': 0} +step=133550 lr=6.000000e-04 loss=2.5023 ce=2.5023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2594391541351797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034268421586602926, 'last_eos_top1': 0} +step=133600 lr=6.000000e-04 loss=2.4895 ce=2.4895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.728749329136917e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029700747691094875, 'last_eos_top1': 0} +step=133650 lr=6.000000e-04 loss=2.4018 ce=2.4018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.500611223396845e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035899830982089043, 'last_eos_top1': 0} +step=133700 lr=6.000000e-04 loss=2.6492 ce=2.6492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.17413435166236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035698330029845238, 'last_eos_top1': 0} +step=133750 lr=6.000000e-04 loss=2.9288 ce=2.9288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.442332728009205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001952870050445199, 'last_eos_top1': 0} +step=133800 lr=6.000000e-04 loss=2.7870 ce=2.7870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.49901597473945e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000471485749585554, 'last_eos_top1': 0} +step=133850 lr=6.000000e-04 loss=2.7503 ce=2.7503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.001618435722776e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.194042311515659e-05, 'last_eos_top1': 0} +step=133900 lr=6.000000e-04 loss=2.0486 ce=2.0486 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.626351594401058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002535897947382182, 'last_eos_top1': 0} +step=133950 lr=6.000000e-04 loss=2.9443 ce=2.9443 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.19731815479463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001190073526231572, 'last_eos_top1': 0} +step=134000 lr=6.000000e-04 loss=2.2729 ce=2.2729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.393914539628895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037715237704105675, 'last_eos_top1': 0} +step=134050 lr=6.000000e-04 loss=2.0871 ce=2.0871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.691198111890117e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029178851400502026, 'last_eos_top1': 0} +step=134100 lr=6.000000e-04 loss=2.5713 ce=2.5713 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.334595466090832e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.442056034458801e-05, 'last_eos_top1': 0} +step=134150 lr=6.000000e-04 loss=2.3897 ce=2.3897 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.182793640618911e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.7460361858829856e-05, 'last_eos_top1': 0} +step=134200 lr=6.000000e-04 loss=2.4238 ce=2.4238 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1768189324357081e-05, 'pos0_bos_top1': 0, 'last_eos_p': 9.119878814090043e-05, 'last_eos_top1': 0} +step=134250 lr=6.000000e-04 loss=2.3944 ce=2.3944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.204555807518773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.7150395908392966e-05, 'last_eos_top1': 0} +step=134300 lr=6.000000e-04 loss=2.6167 ce=2.6167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.294444357175962e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.4238770745578222e-05, 'last_eos_top1': 0} +step=134350 lr=6.000000e-04 loss=2.2166 ce=2.2166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.301897206460126e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.1339640776859596e-05, 'last_eos_top1': 0} +step=134400 lr=6.000000e-04 loss=2.4582 ce=2.4582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.526310582761653e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044921672088094056, 'last_eos_top1': 0} +step=134450 lr=6.000000e-04 loss=2.6288 ce=2.6288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.804208507877775e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006819750415161252, 'last_eos_top1': 0} +step=134500 lr=6.000000e-04 loss=2.3926 ce=2.3926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.816985554294661e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000732855754904449, 'last_eos_top1': 0} +step=134550 lr=6.000000e-04 loss=2.3746 ce=2.3746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.134637947572628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018736333004198968, 'last_eos_top1': 0} +step=134600 lr=6.000000e-04 loss=2.3158 ce=2.3158 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7959049500859692e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.9484395023901016e-05, 'last_eos_top1': 0} +step=134650 lr=6.000000e-04 loss=2.6188 ce=2.6188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2291003460850334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.941384784411639e-05, 'last_eos_top1': 0} +step=134700 lr=6.000000e-04 loss=2.7931 ce=2.7931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.794199756157468e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022314407397061586, 'last_eos_top1': 0} +step=134750 lr=6.000000e-04 loss=2.5690 ce=2.5690 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.710537607228616e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004864720394834876, 'last_eos_top1': 0} +step=134800 lr=6.000000e-04 loss=2.4737 ce=2.4737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.073488526046276e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001838592579588294, 'last_eos_top1': 0} +step=134850 lr=6.000000e-04 loss=2.9658 ce=2.9658 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4975033738883212e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.556891220388934e-05, 'last_eos_top1': 0} +step=134900 lr=6.000000e-04 loss=2.5157 ce=2.5157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.969651970692212e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.21918508032104e-05, 'last_eos_top1': 0} +step=134950 lr=6.000000e-04 loss=2.8785 ce=2.8785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.071141903405078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.241730054374784e-05, 'last_eos_top1': 0} +step=135000 lr=6.000000e-04 loss=2.4639 ce=2.4639 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9251065168646164e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.3004516404180322e-05, 'last_eos_top1': 0} +step=135050 lr=6.000000e-04 loss=2.5043 ce=2.5043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.585333954310045e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.666795080585871e-05, 'last_eos_top1': 0} +step=135100 lr=6.000000e-04 loss=2.5785 ce=2.5785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.081091436906718e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002566119364928454, 'last_eos_top1': 0} +step=135150 lr=6.000000e-04 loss=2.5319 ce=2.5319 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.697520580521086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005090502090752125, 'last_eos_top1': 0} +step=135200 lr=6.000000e-04 loss=2.8046 ce=2.8046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.51509561369312e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027704317471943796, 'last_eos_top1': 0} +step=135250 lr=6.000000e-04 loss=2.6097 ce=2.6097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1685402771254303e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.445673352573067e-05, 'last_eos_top1': 0} +step=135300 lr=6.000000e-04 loss=2.3296 ce=2.3296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.841469151666388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.984759915620089e-05, 'last_eos_top1': 0} +step=135350 lr=6.000000e-04 loss=2.1571 ce=2.1571 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.780760147899855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.257208148483187e-05, 'last_eos_top1': 0} +step=135400 lr=6.000000e-04 loss=2.5063 ce=2.5063 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.440188604348805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.851046444149688e-05, 'last_eos_top1': 0} +step=135450 lr=6.000000e-04 loss=2.3679 ce=2.3679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.131031801080098e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013026539818383753, 'last_eos_top1': 0} +step=135500 lr=6.000000e-04 loss=2.1531 ce=2.1531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.108782832190627e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.68040884193033e-05, 'last_eos_top1': 0} +step=135550 lr=6.000000e-04 loss=2.4685 ce=2.4685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.308270950379665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017493347695562989, 'last_eos_top1': 0} +step=135600 lr=6.000000e-04 loss=2.3122 ce=2.3122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5933559552649967e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007492388249374926, 'last_eos_top1': 0} +step=135650 lr=6.000000e-04 loss=2.2669 ce=2.2669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4575052773579955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009908752981573343, 'last_eos_top1': 0} +step=135700 lr=6.000000e-04 loss=2.0461 ce=2.0461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.764307833393104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009165193187072873, 'last_eos_top1': 0} +step=135750 lr=6.000000e-04 loss=2.4030 ce=2.4030 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.661006071022712e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007411232800222933, 'last_eos_top1': 0} +step=135800 lr=6.000000e-04 loss=2.7884 ce=2.7884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.973660674702842e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024309544824063778, 'last_eos_top1': 0} +step=135850 lr=6.000000e-04 loss=2.4583 ce=2.4583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.656027956822072e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004339379956945777, 'last_eos_top1': 0} +step=135900 lr=6.000000e-04 loss=2.6290 ce=2.6290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5352037482662126e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.7359110137913376e-05, 'last_eos_top1': 0} +step=135950 lr=6.000000e-04 loss=2.7161 ce=2.7161 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.983935468771961e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.325807771645486e-05, 'last_eos_top1': 0} +step=136000 lr=6.000000e-04 loss=2.0894 ce=2.0894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.086957576670102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018210551934316754, 'last_eos_top1': 0} +step=136050 lr=6.000000e-04 loss=2.4767 ce=2.4767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1314953048422467e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016403477638959885, 'last_eos_top1': 0} +step=136100 lr=6.000000e-04 loss=2.0836 ce=2.0836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1008107321686111e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011998130939900875, 'last_eos_top1': 0} +step=136150 lr=6.000000e-04 loss=2.1562 ce=2.1562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.538015895785065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009283357649110258, 'last_eos_top1': 0} +step=136200 lr=6.000000e-04 loss=1.6512 ce=1.6512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2176420720352326e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043029163498431444, 'last_eos_top1': 0} +step=136250 lr=6.000000e-04 loss=2.5391 ce=2.5391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0979919099772815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024465820752084255, 'last_eos_top1': 0} +step=136300 lr=6.000000e-04 loss=2.4190 ce=2.4190 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7185078579350375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006962062325328588, 'last_eos_top1': 0} +step=136350 lr=6.000000e-04 loss=2.2995 ce=2.2995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.366326149669476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007511230651289225, 'last_eos_top1': 0} +step=136400 lr=6.000000e-04 loss=3.1760 ce=3.1760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.851561698160367e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025729008484631777, 'last_eos_top1': 0} +step=136450 lr=6.000000e-04 loss=2.6035 ce=2.6035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.005362486874219e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026742350310087204, 'last_eos_top1': 0} +step=136500 lr=6.000000e-04 loss=2.4227 ce=2.4227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.949988811451476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018845191225409508, 'last_eos_top1': 0} +step=136550 lr=6.000000e-04 loss=2.4815 ce=2.4815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.154968799710332e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038822193164378405, 'last_eos_top1': 0} +step=136600 lr=6.000000e-04 loss=1.9660 ce=1.9660 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1576793187705334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002669969107955694, 'last_eos_top1': 0} +step=136650 lr=6.000000e-04 loss=2.4669 ce=2.4669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8425075621926226e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.208535342011601e-05, 'last_eos_top1': 0} +step=136700 lr=6.000000e-04 loss=2.4372 ce=2.4372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7817875383771025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.940495561342686e-05, 'last_eos_top1': 0} +step=136750 lr=6.000000e-04 loss=2.7488 ce=2.7488 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4509649847459514e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005201073945499957, 'last_eos_top1': 0} +step=136800 lr=6.000000e-04 loss=2.4109 ce=2.4109 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.002406967629213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007641261327080429, 'last_eos_top1': 0} +step=136850 lr=6.000000e-04 loss=2.6741 ce=2.6741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.760939191328362e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007527651032432914, 'last_eos_top1': 0} +step=136900 lr=6.000000e-04 loss=2.4476 ce=2.4476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3104236081271665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006180406780913472, 'last_eos_top1': 0} +step=136950 lr=6.000000e-04 loss=2.2252 ce=2.2252 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4477280931023415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006234022439457476, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2200000 rows=2191809 buffered=8191 part=openwebtext-t5 +step=137000 lr=6.000000e-04 loss=3.1600 ce=3.1600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.467826667882036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000653821334708482, 'last_eos_top1': 0} +step=137050 lr=6.000000e-04 loss=2.0391 ce=2.0391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3348790162126534e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040970370173454285, 'last_eos_top1': 0} +step=137100 lr=6.000000e-04 loss=3.2129 ce=3.2129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.957349902833812e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007777918362990022, 'last_eos_top1': 0} +step=137150 lr=6.000000e-04 loss=2.2884 ce=2.2884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.551745405478869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003378466935828328, 'last_eos_top1': 0} +step=137200 lr=6.000000e-04 loss=2.2860 ce=2.2860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.152423106366768e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004185866564512253, 'last_eos_top1': 0} +step=137250 lr=6.000000e-04 loss=2.9585 ce=2.9585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.156762886646902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022411560348700732, 'last_eos_top1': 0} +step=137300 lr=6.000000e-04 loss=2.3499 ce=2.3499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2069156077341177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002002582623390481, 'last_eos_top1': 0} +step=137350 lr=6.000000e-04 loss=2.3837 ce=2.3837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.153928784944583e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005036493530496955, 'last_eos_top1': 0} +step=137400 lr=6.000000e-04 loss=2.9759 ce=2.9759 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2364940630504861e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006258381763473153, 'last_eos_top1': 0} +step=137450 lr=6.000000e-04 loss=2.2800 ce=2.2800 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0595767889753915e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010604863055050373, 'last_eos_top1': 0} +step=137500 lr=6.000000e-04 loss=1.9853 ce=1.9853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.659550647076685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007873867871239781, 'last_eos_top1': 0} +step=137550 lr=6.000000e-04 loss=2.4736 ce=2.4736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.830779289477505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008331930730491877, 'last_eos_top1': 0} +step=137600 lr=6.000000e-04 loss=2.2069 ce=2.2069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.682225724332966e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000792677397839725, 'last_eos_top1': 0} +step=137650 lr=6.000000e-04 loss=2.6664 ce=2.6664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.323828994121868e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000955721247009933, 'last_eos_top1': 0} +step=137700 lr=6.000000e-04 loss=2.6016 ce=2.6016 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.390058165881783e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006935769924893975, 'last_eos_top1': 0} +step=137750 lr=6.000000e-04 loss=2.4059 ce=2.4059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8407100697004353e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007766092894598842, 'last_eos_top1': 0} +step=137800 lr=6.000000e-04 loss=2.6875 ce=2.6875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.04303944176354e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012327830772846937, 'last_eos_top1': 0} +step=137850 lr=6.000000e-04 loss=2.2395 ce=2.2395 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.727824610497919e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013983792159706354, 'last_eos_top1': 0} +step=137900 lr=6.000000e-04 loss=2.7079 ce=2.7079 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5548323517577955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011799067724496126, 'last_eos_top1': 0} +step=137950 lr=6.000000e-04 loss=2.3066 ce=2.3066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.325644345546607e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018318924121558666, 'last_eos_top1': 0} +step=138000 lr=6.000000e-04 loss=2.7357 ce=2.7357 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.43197337113088e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010198978707194328, 'last_eos_top1': 0} +step=138050 lr=6.000000e-04 loss=2.5896 ce=2.5896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2744493435311597e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001113811507821083, 'last_eos_top1': 0} +step=138100 lr=6.000000e-04 loss=2.2339 ce=2.2339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6374356164305937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001287104794755578, 'last_eos_top1': 0} +step=138150 lr=6.000000e-04 loss=2.4723 ce=2.4723 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.205817731417483e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001175531418994069, 'last_eos_top1': 0} +step=138200 lr=6.000000e-04 loss=2.3334 ce=2.3334 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2706165004346985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020989631302654743, 'last_eos_top1': 0} +step=138250 lr=6.000000e-04 loss=2.3187 ce=2.3187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.168121101130964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001078432658687234, 'last_eos_top1': 0} +step=138300 lr=6.000000e-04 loss=3.1669 ce=3.1669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.713143885557656e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007205415749922395, 'last_eos_top1': 0} +step=138350 lr=6.000000e-04 loss=2.7085 ce=2.7085 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.851154476957163e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008497733506374061, 'last_eos_top1': 0} +step=138400 lr=6.000000e-04 loss=2.8824 ce=2.8824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.125682607991621e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009752605692483485, 'last_eos_top1': 0} +step=138450 lr=6.000000e-04 loss=2.4489 ce=2.4489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.416200343053788e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007721713045611978, 'last_eos_top1': 0} +step=138500 lr=6.000000e-04 loss=2.5305 ce=2.5305 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.423218120384263e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010446814121678472, 'last_eos_top1': 0} +step=138550 lr=6.000000e-04 loss=2.7350 ce=2.7350 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.503424861468375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013041668571531773, 'last_eos_top1': 0} +step=138600 lr=6.000000e-04 loss=2.1844 ce=2.1844 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.802761734230444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007630347972735763, 'last_eos_top1': 0} +step=138650 lr=6.000000e-04 loss=2.1443 ce=2.1443 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.392617145436816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006729424931108952, 'last_eos_top1': 0} +step=138700 lr=6.000000e-04 loss=2.6512 ce=2.6512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.433713679143693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010252429638057947, 'last_eos_top1': 0} +step=138750 lr=6.000000e-04 loss=2.8843 ce=2.8843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.01751822209917e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007391278631985188, 'last_eos_top1': 0} +step=138800 lr=6.000000e-04 loss=1.8376 ce=1.8376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2441063265432604e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009836273966357112, 'last_eos_top1': 0} +step=138850 lr=6.000000e-04 loss=2.3655 ce=2.3655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.371514367245254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012694504112005234, 'last_eos_top1': 0} +step=138900 lr=6.000000e-04 loss=2.8162 ce=2.8162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6089687051135115e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001210373593494296, 'last_eos_top1': 0} +step=138950 lr=6.000000e-04 loss=2.5611 ce=2.5611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.755638762318995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008087716996669769, 'last_eos_top1': 0} +step=139000 lr=6.000000e-04 loss=2.8779 ce=2.8779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.769798917754088e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007329643703997135, 'last_eos_top1': 0} +step=139050 lr=6.000000e-04 loss=2.8679 ce=2.8679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6373836590209976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006372850621119142, 'last_eos_top1': 0} +step=139100 lr=6.000000e-04 loss=2.4073 ce=2.4073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.344841050216928e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006017801351845264, 'last_eos_top1': 0} +step=139150 lr=6.000000e-04 loss=2.7047 ce=2.7047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.263173312006984e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008810824947431684, 'last_eos_top1': 0} +step=139200 lr=6.000000e-04 loss=2.2086 ce=2.2086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6276585382875055e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016134199686348438, 'last_eos_top1': 0} +step=139250 lr=6.000000e-04 loss=2.7373 ce=2.7373 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.988855835108552e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000708458130247891, 'last_eos_top1': 0} +step=139300 lr=6.000000e-04 loss=2.4550 ce=2.4550 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.513276043231599e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008151334477588534, 'last_eos_top1': 0} +step=139350 lr=6.000000e-04 loss=3.0129 ce=3.0129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4488294911861885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000907000619918108, 'last_eos_top1': 0} +step=139400 lr=6.000000e-04 loss=2.1417 ce=2.1417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.471479890824412e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021698177442885935, 'last_eos_top1': 0} +step=139450 lr=6.000000e-04 loss=2.2379 ce=2.2379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.812421477457974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005988131160847843, 'last_eos_top1': 0} +step=139500 lr=6.000000e-04 loss=2.4715 ce=2.4715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0680879465071484e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002573479723650962, 'last_eos_top1': 0} +step=139550 lr=6.000000e-04 loss=2.3431 ce=2.3431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.910063115763478e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006392800714820623, 'last_eos_top1': 0} +step=139600 lr=6.000000e-04 loss=2.3542 ce=2.3542 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.32299873662123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015754306223243475, 'last_eos_top1': 0} +step=139650 lr=6.000000e-04 loss=2.5586 ce=2.5586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2295579330821056e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001643135561607778, 'last_eos_top1': 0} +step=139700 lr=6.000000e-04 loss=2.2397 ce=2.2397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.777999831479974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020025488920509815, 'last_eos_top1': 0} +step=139750 lr=6.000000e-04 loss=2.7446 ce=2.7446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.394502866489347e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015101239550858736, 'last_eos_top1': 0} +step=139800 lr=6.000000e-04 loss=3.0189 ce=3.0189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1672178718290525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011539740953594446, 'last_eos_top1': 0} +step=139850 lr=6.000000e-04 loss=2.2558 ce=2.2558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.866019925335422e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012172963470220566, 'last_eos_top1': 0} +step=139900 lr=6.000000e-04 loss=2.7576 ce=2.7576 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.732350210659206e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001768191228620708, 'last_eos_top1': 0} +step=139950 lr=6.000000e-04 loss=2.9517 ce=2.9517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.650052763579879e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009689326398074627, 'last_eos_top1': 0} +step=140000 lr=6.000000e-04 loss=2.6029 ce=2.6029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5222010915749706e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011061773402616382, 'last_eos_top1': 0} +step=140050 lr=6.000000e-04 loss=2.1174 ce=2.1174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.504940418177284e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010252560023218393, 'last_eos_top1': 0} +step=140100 lr=6.000000e-04 loss=2.5663 ce=2.5663 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1482697926985566e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024429108016192913, 'last_eos_top1': 0} +step=140150 lr=6.000000e-04 loss=2.5594 ce=2.5594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7788440749864094e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017125974409282207, 'last_eos_top1': 0} +step=140200 lr=6.000000e-04 loss=2.5616 ce=2.5616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2616000023845118e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014809883432462811, 'last_eos_top1': 0} +step=140250 lr=6.000000e-04 loss=2.4158 ce=2.4158 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8672776604944374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001104760798625648, 'last_eos_top1': 0} +step=140300 lr=6.000000e-04 loss=3.1322 ce=3.1322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3925076675368473e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007256768876686692, 'last_eos_top1': 0} +step=140350 lr=6.000000e-04 loss=2.3175 ce=2.3175 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6153873048097012e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015441247960552573, 'last_eos_top1': 0} +step=140400 lr=6.000000e-04 loss=2.4268 ce=2.4268 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1351730790920556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001831317087635398, 'last_eos_top1': 0} +step=140450 lr=6.000000e-04 loss=2.5335 ce=2.5335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.378760761232115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020551015622913837, 'last_eos_top1': 0} +step=140500 lr=6.000000e-04 loss=2.4297 ce=2.4297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2444099815620575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018106207717210054, 'last_eos_top1': 0} +step=140550 lr=6.000000e-04 loss=2.4417 ce=2.4417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.859463504340965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016242393758147955, 'last_eos_top1': 0} +step=140600 lr=6.000000e-04 loss=2.1602 ce=2.1602 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1388904062623624e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008844573749229312, 'last_eos_top1': 0} +step=140650 lr=6.000000e-04 loss=2.2805 ce=2.2805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2702698717912426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010102340020239353, 'last_eos_top1': 0} +step=140700 lr=6.000000e-04 loss=2.6486 ce=2.6486 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.202459538442781e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007517731282860041, 'last_eos_top1': 0} +step=140750 lr=6.000000e-04 loss=2.4185 ce=2.4185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5891322366078384e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003166375099681318, 'last_eos_top1': 0} +step=140800 lr=6.000000e-04 loss=2.3069 ce=2.3069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5322918847668916e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002537975087761879, 'last_eos_top1': 0} +step=140850 lr=6.000000e-04 loss=2.3631 ce=2.3631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8323895574212656e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002453327761031687, 'last_eos_top1': 0} +step=140900 lr=6.000000e-04 loss=1.9195 ce=1.9195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7360249532648595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002845374692697078, 'last_eos_top1': 0} +step=140950 lr=6.000000e-04 loss=2.0946 ce=2.0946 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5117749373748666e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.908979462925345e-05, 'last_eos_top1': 0} +step=141000 lr=6.000000e-04 loss=2.2532 ce=2.2532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5834092411969323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.699954686453566e-05, 'last_eos_top1': 0} +step=141050 lr=6.000000e-04 loss=2.4972 ce=2.4972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0509210116870236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015436131798196584, 'last_eos_top1': 0} +step=141100 lr=6.000000e-04 loss=2.6067 ce=2.6067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.087052735078032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00025273242499679327, 'last_eos_top1': 0} +step=141150 lr=6.000000e-04 loss=2.5041 ce=2.5041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.54125529358862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007202156120911241, 'last_eos_top1': 0} +step=141200 lr=6.000000e-04 loss=2.0438 ce=2.0438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.597569618956186e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003259047807659954, 'last_eos_top1': 0} +step=141250 lr=6.000000e-04 loss=2.4812 ce=2.4812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8798449420719407e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000863511289935559, 'last_eos_top1': 0} +step=141300 lr=6.000000e-04 loss=2.5393 ce=2.5393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9874918254790828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031454031704925, 'last_eos_top1': 0} +step=141350 lr=6.000000e-04 loss=2.0519 ce=2.0519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.469130660960218e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042641727486625314, 'last_eos_top1': 0} +step=141400 lr=6.000000e-04 loss=2.6493 ce=2.6493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4362662972853286e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006575739826075733, 'last_eos_top1': 0} +step=141450 lr=6.000000e-04 loss=2.4625 ce=2.4625 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.981899110134691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001433963654562831, 'last_eos_top1': 0} +step=141500 lr=6.000000e-04 loss=1.9111 ce=1.9111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2962299226492178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005980085115879774, 'last_eos_top1': 0} +step=141550 lr=6.000000e-04 loss=2.2501 ce=2.2501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.79570348438574e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014030038437340409, 'last_eos_top1': 0} +step=141600 lr=6.000000e-04 loss=2.5110 ce=2.5110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.61811168861459e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010069741983897984, 'last_eos_top1': 0} +step=141650 lr=6.000000e-04 loss=2.4647 ce=2.4647 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.670816795420251e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.8885056963190436e-05, 'last_eos_top1': 0} +step=141700 lr=6.000000e-04 loss=2.4554 ce=2.4554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.358410362532595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.499233571346849e-05, 'last_eos_top1': 0} +step=141750 lr=6.000000e-04 loss=2.6340 ce=2.6340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.293372734944569e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001425564259989187, 'last_eos_top1': 0} +step=141800 lr=6.000000e-04 loss=2.7688 ce=2.7688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.989745088503696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045270068221725523, 'last_eos_top1': 0} +step=141850 lr=6.000000e-04 loss=2.3196 ce=2.3196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.511277312412858e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009845234453678131, 'last_eos_top1': 0} +step=141900 lr=6.000000e-04 loss=2.6263 ce=2.6263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7449626688612625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047521002124994993, 'last_eos_top1': 0} +step=141950 lr=6.000000e-04 loss=2.6803 ce=2.6803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.463145731075201e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005367131670936942, 'last_eos_top1': 0} +step=142000 lr=6.000000e-04 loss=2.3991 ce=2.3991 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.470679414225742e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007581511745229363, 'last_eos_top1': 0} +step=142050 lr=6.000000e-04 loss=2.3929 ce=2.3929 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3674659789539874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007862566853873432, 'last_eos_top1': 0} +step=142100 lr=6.000000e-04 loss=2.4154 ce=2.4154 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.62805598747218e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016128021525219083, 'last_eos_top1': 0} +step=142150 lr=6.000000e-04 loss=2.7507 ce=2.7507 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.766121724562254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000494504754897207, 'last_eos_top1': 0} +step=142200 lr=6.000000e-04 loss=2.4065 ce=2.4065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.84573331757565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039554282557219267, 'last_eos_top1': 0} +step=142250 lr=6.000000e-04 loss=2.6939 ce=2.6939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.597566996584646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000196368113392964, 'last_eos_top1': 0} +step=142300 lr=6.000000e-04 loss=2.2779 ce=2.2779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.459418960323092e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.32417024159804e-05, 'last_eos_top1': 0} +step=142350 lr=6.000000e-04 loss=2.8496 ce=2.8496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5327527661575004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019140027870889753, 'last_eos_top1': 0} +step=142400 lr=6.000000e-04 loss=2.0578 ce=2.0578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.289320587413386e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026208284543827176, 'last_eos_top1': 0} +step=142450 lr=6.000000e-04 loss=2.5494 ce=2.5494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8423832999542356e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007510639843530953, 'last_eos_top1': 0} +step=142500 lr=6.000000e-04 loss=2.3644 ce=2.3644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.926583980908617e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006203814991749823, 'last_eos_top1': 0} +step=142550 lr=6.000000e-04 loss=2.7005 ce=2.7005 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.283378075342625e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002621900930535048, 'last_eos_top1': 0} +step=142600 lr=6.000000e-04 loss=2.7017 ce=2.7017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.314084203040693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001375990454107523, 'last_eos_top1': 0} +step=142650 lr=6.000000e-04 loss=2.8823 ce=2.8823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.906812591798371e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009263245156034827, 'last_eos_top1': 0} +step=142700 lr=6.000000e-04 loss=2.4938 ce=2.4938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.124392257333966e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012073174584656954, 'last_eos_top1': 0} +step=142750 lr=6.000000e-04 loss=2.2275 ce=2.2275 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.484142431872897e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002680812031030655, 'last_eos_top1': 0} +step=142800 lr=6.000000e-04 loss=2.7322 ce=2.7322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.253744009474758e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035404780646786094, 'last_eos_top1': 0} +step=142850 lr=6.000000e-04 loss=2.9167 ce=2.9167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.086370947537944e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004931208095513284, 'last_eos_top1': 0} +step=142900 lr=6.000000e-04 loss=2.6162 ce=2.6162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.696254786220379e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035908143036067486, 'last_eos_top1': 0} +step=142950 lr=6.000000e-04 loss=2.7598 ce=2.7598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7128406802366953e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008578522829338908, 'last_eos_top1': 0} +step=143000 lr=6.000000e-04 loss=2.7972 ce=2.7972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.343034899851773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000845403759740293, 'last_eos_top1': 0} +step=143050 lr=6.000000e-04 loss=2.5974 ce=2.5974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.696342774783261e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00128004583530128, 'last_eos_top1': 0} +step=143100 lr=6.000000e-04 loss=2.5846 ce=2.5846 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.440553417836782e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001868540421128273, 'last_eos_top1': 0} +step=143150 lr=6.000000e-04 loss=2.0572 ce=2.0572 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.899711140955333e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009717364446260035, 'last_eos_top1': 0} +step=143200 lr=6.000000e-04 loss=2.5992 ce=2.5992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.834224571619416e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006628717528656125, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2300000 rows=2291809 buffered=8191 part=openwebtext-t5 +step=143250 lr=6.000000e-04 loss=2.4634 ce=2.4634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.816021828446537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020290843676775694, 'last_eos_top1': 0} +step=143300 lr=6.000000e-04 loss=3.0782 ce=3.0782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.120692094351398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019799876026809216, 'last_eos_top1': 0} +step=143350 lr=6.000000e-04 loss=2.6371 ce=2.6371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.92432797930087e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019822148606181145, 'last_eos_top1': 0} +step=143400 lr=6.000000e-04 loss=2.3098 ce=2.3098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.616902853944339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011437182547524571, 'last_eos_top1': 0} +step=143450 lr=6.000000e-04 loss=2.4447 ce=2.4447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.826500116905663e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018503250321373343, 'last_eos_top1': 0} +step=143500 lr=6.000000e-04 loss=2.0768 ce=2.0768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6861724765913095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002136091934517026, 'last_eos_top1': 0} +step=143550 lr=6.000000e-04 loss=2.2551 ce=2.2551 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.334686077025253e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000913016265258193, 'last_eos_top1': 0} +step=143600 lr=6.000000e-04 loss=2.7014 ce=2.7014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.681427865056321e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004143051046412438, 'last_eos_top1': 0} +step=143650 lr=6.000000e-04 loss=2.8287 ce=2.8287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.124227416468784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004904603702016175, 'last_eos_top1': 0} +step=143700 lr=6.000000e-04 loss=2.6900 ce=2.6900 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.186813723412342e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018109020311385393, 'last_eos_top1': 0} +step=143750 lr=6.000000e-04 loss=2.3732 ce=2.3732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4265613117458997e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002939308760687709, 'last_eos_top1': 0} +step=143800 lr=6.000000e-04 loss=2.4071 ce=2.4071 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.222587343363557e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007049036212265491, 'last_eos_top1': 0} +step=143850 lr=6.000000e-04 loss=2.6904 ce=2.6904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.361866664723493e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004327117349021137, 'last_eos_top1': 0} +step=143900 lr=6.000000e-04 loss=2.5742 ce=2.5742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.904419827449601e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.821874169167131e-05, 'last_eos_top1': 0} +step=143950 lr=6.000000e-04 loss=2.5094 ce=2.5094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6323751778109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.951867832569405e-05, 'last_eos_top1': 0} +step=144000 lr=6.000000e-04 loss=2.1421 ce=2.1421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2450529538909905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020773260621353984, 'last_eos_top1': 0} +step=144050 lr=6.000000e-04 loss=2.6677 ce=2.6677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.10179200116545e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.3957636333070695e-05, 'last_eos_top1': 0} +step=144100 lr=6.000000e-04 loss=2.4093 ce=2.4093 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.628620783681981e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.8373246146366e-05, 'last_eos_top1': 0} +step=144150 lr=6.000000e-04 loss=2.7978 ce=2.7978 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.254001618799521e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033941533183678985, 'last_eos_top1': 0} +step=144200 lr=6.000000e-04 loss=2.0544 ce=2.0544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.83616088583949e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001767326903063804, 'last_eos_top1': 0} +step=144250 lr=6.000000e-04 loss=2.4471 ce=2.4471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.339242761692731e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002934453950729221, 'last_eos_top1': 0} +step=144300 lr=6.000000e-04 loss=2.1262 ce=2.1262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3900350899784826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009496249258518219, 'last_eos_top1': 0} +step=144350 lr=6.000000e-04 loss=2.4178 ce=2.4178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.032473952975124e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013193866470828652, 'last_eos_top1': 0} +step=144400 lr=6.000000e-04 loss=2.0439 ce=2.0439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.124745267588878e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012960531748831272, 'last_eos_top1': 0} +step=144450 lr=6.000000e-04 loss=2.5565 ce=2.5565 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.39212646294618e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001091664656996727, 'last_eos_top1': 0} +step=144500 lr=6.000000e-04 loss=2.2396 ce=2.2396 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3863101382157765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010772529058158398, 'last_eos_top1': 0} +step=144550 lr=6.000000e-04 loss=2.9555 ce=2.9555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8975570128532127e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000993664376437664, 'last_eos_top1': 0} +step=144600 lr=6.000000e-04 loss=2.3487 ce=2.3487 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.424529263109434e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014019556110724807, 'last_eos_top1': 0} +step=144650 lr=6.000000e-04 loss=2.5250 ce=2.5250 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.609537765849382e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005567627376876771, 'last_eos_top1': 0} +step=144700 lr=6.000000e-04 loss=2.5563 ce=2.5563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.04956141312141e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001315029221586883, 'last_eos_top1': 0} +step=144750 lr=6.000000e-04 loss=2.5082 ce=2.5082 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.013730621314608e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018937999848276377, 'last_eos_top1': 0} +step=144800 lr=6.000000e-04 loss=2.4275 ce=2.4275 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.388779987290036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001497626886703074, 'last_eos_top1': 0} +step=144850 lr=6.000000e-04 loss=3.1518 ce=3.1518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3686849292280385e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001780959777534008, 'last_eos_top1': 0} +step=144900 lr=6.000000e-04 loss=2.4823 ce=2.4823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.11184972411138e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001054257620126009, 'last_eos_top1': 0} +step=144950 lr=6.000000e-04 loss=2.7716 ce=2.7716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.648883754678536e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008098966209217906, 'last_eos_top1': 0} +step=145000 lr=6.000000e-04 loss=2.5267 ce=2.5267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4335005188477226e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010544485412538052, 'last_eos_top1': 0} +step=145050 lr=6.000000e-04 loss=2.3701 ce=2.3701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.384837782505201e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017871293239295483, 'last_eos_top1': 0} +step=145100 lr=6.000000e-04 loss=2.8771 ce=2.8771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5612185961799696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017239925218746066, 'last_eos_top1': 0} +step=145150 lr=6.000000e-04 loss=2.1335 ce=2.1335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.374325726530515e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002317981328815222, 'last_eos_top1': 0} +step=145200 lr=6.000000e-04 loss=2.4143 ce=2.4143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.406107447925024e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001407828414812684, 'last_eos_top1': 0} +step=145250 lr=6.000000e-04 loss=2.3063 ce=2.3063 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.522001523175277e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004540613736025989, 'last_eos_top1': 0} +step=145300 lr=6.000000e-04 loss=2.6200 ce=2.6200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.084870281280018e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007653791108168662, 'last_eos_top1': 0} +step=145350 lr=6.000000e-04 loss=2.6590 ce=2.6590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7056516955781262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003094780840910971, 'last_eos_top1': 0} +step=145400 lr=6.000000e-04 loss=2.5620 ce=2.5620 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.617469590404653e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007869633263908327, 'last_eos_top1': 0} +step=145450 lr=6.000000e-04 loss=2.6270 ce=2.6270 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5550816644681618e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005550330970436335, 'last_eos_top1': 0} +step=145500 lr=6.000000e-04 loss=2.0164 ce=2.0164 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.9272206271998584e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014541159616783261, 'last_eos_top1': 0} +step=145550 lr=6.000000e-04 loss=2.2896 ce=2.2896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3142336431192234e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006638007471337914, 'last_eos_top1': 0} +step=145600 lr=6.000000e-04 loss=2.8094 ce=2.8094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.179119282023748e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.030083750374615e-05, 'last_eos_top1': 0} +step=145650 lr=6.000000e-04 loss=2.9086 ce=2.9086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4474038582411595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.2821618990274146e-05, 'last_eos_top1': 0} +step=145700 lr=6.000000e-04 loss=2.2721 ce=2.2721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2372595342167187e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019966711988672614, 'last_eos_top1': 0} +step=145750 lr=6.000000e-04 loss=2.7515 ce=2.7515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3171515951835318e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.0943527235649526e-05, 'last_eos_top1': 0} +step=145800 lr=6.000000e-04 loss=2.4864 ce=2.4864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.305039965402102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.882657306850888e-05, 'last_eos_top1': 0} +step=145850 lr=6.000000e-04 loss=2.5763 ce=2.5763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.231141247146297e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.5967183319153264e-05, 'last_eos_top1': 0} +step=145900 lr=6.000000e-04 loss=2.3741 ce=2.3741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.361375547683565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.773649637470953e-05, 'last_eos_top1': 0} +step=145950 lr=6.000000e-04 loss=3.1633 ce=3.1633 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.971272109192796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.236502228304744e-05, 'last_eos_top1': 0} +step=146000 lr=6.000000e-04 loss=2.0823 ce=2.0823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7511001715320162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 1.5387424355139956e-05, 'last_eos_top1': 0} +step=146050 lr=6.000000e-04 loss=2.5628 ce=2.5628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.571281922631897e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.1177572964224964e-05, 'last_eos_top1': 0} +step=146100 lr=6.000000e-04 loss=2.9974 ce=2.9974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.118707693123724e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.530623745173216e-05, 'last_eos_top1': 0} +step=146150 lr=6.000000e-04 loss=2.1703 ce=2.1703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.897664313029964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006714175106026232, 'last_eos_top1': 0} +step=146200 lr=6.000000e-04 loss=2.8827 ce=2.8827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.774345587880816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022759611601941288, 'last_eos_top1': 0} +step=146250 lr=6.000000e-04 loss=2.1211 ce=2.1211 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.937082278251182e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.828824552940205e-05, 'last_eos_top1': 0} +step=146300 lr=6.000000e-04 loss=3.0187 ce=3.0187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5751262253616005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00197618268430233, 'last_eos_top1': 0} +step=146350 lr=6.000000e-04 loss=2.6408 ce=2.6408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.788872032193467e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015632237773388624, 'last_eos_top1': 0} +step=146400 lr=6.000000e-04 loss=2.7214 ce=2.7214 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.878963525494328e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012450956273823977, 'last_eos_top1': 0} +step=146450 lr=6.000000e-04 loss=2.6274 ce=2.6274 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.776216772166663e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007650585612282157, 'last_eos_top1': 0} +step=146500 lr=6.000000e-04 loss=2.6203 ce=2.6203 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.541152859223075e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019164965488016605, 'last_eos_top1': 0} +step=146550 lr=6.000000e-04 loss=2.5340 ce=2.5340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.230525519233197e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024679708294570446, 'last_eos_top1': 0} +step=146600 lr=6.000000e-04 loss=2.2810 ce=2.2810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2980973375961185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002139634918421507, 'last_eos_top1': 0} +step=146650 lr=6.000000e-04 loss=2.6832 ce=2.6832 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6423789424588904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005452888435684144, 'last_eos_top1': 0} +step=146700 lr=6.000000e-04 loss=2.8732 ce=2.8732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.292764515412273e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001762612082529813, 'last_eos_top1': 0} +step=146750 lr=6.000000e-04 loss=2.2450 ce=2.2450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.778084192163078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.835997141432017e-05, 'last_eos_top1': 0} +step=146800 lr=6.000000e-04 loss=2.7096 ce=2.7096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1298557183181401e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014136848039925098, 'last_eos_top1': 0} +step=146850 lr=6.000000e-04 loss=2.3255 ce=2.3255 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4815980193816358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000773690058849752, 'last_eos_top1': 0} +step=146900 lr=6.000000e-04 loss=2.4387 ce=2.4387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.474128450558055e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036669441033154726, 'last_eos_top1': 0} +step=146950 lr=6.000000e-04 loss=2.3075 ce=2.3075 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.083795607177308e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013460451737046242, 'last_eos_top1': 0} +step=147000 lr=6.000000e-04 loss=3.1009 ce=3.1009 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6649121334921801e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040768837789073586, 'last_eos_top1': 0} +step=147050 lr=6.000000e-04 loss=2.6480 ce=2.6480 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2974944638699526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027501798467710614, 'last_eos_top1': 0} +step=147100 lr=6.000000e-04 loss=2.5288 ce=2.5288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.84463440545369e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004811955150216818, 'last_eos_top1': 0} +step=147150 lr=6.000000e-04 loss=2.7720 ce=2.7720 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4538033913995605e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022452399134635925, 'last_eos_top1': 0} +step=147200 lr=6.000000e-04 loss=2.7592 ce=2.7592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.402339077889337e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021840212866663933, 'last_eos_top1': 0} +step=147250 lr=6.000000e-04 loss=2.4861 ce=2.4861 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0718276775587583e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019475590670481324, 'last_eos_top1': 0} +step=147300 lr=6.000000e-04 loss=2.1990 ce=2.1990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.689493718818994e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002107946202158928, 'last_eos_top1': 0} +step=147350 lr=6.000000e-04 loss=2.6589 ce=2.6589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1010392780881375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022591743618249893, 'last_eos_top1': 0} +step=147400 lr=6.000000e-04 loss=2.3864 ce=2.3864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.006611106888158e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014059662353247404, 'last_eos_top1': 0} +step=147450 lr=6.000000e-04 loss=2.2279 ce=2.2279 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2460367290477734e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024598410818725824, 'last_eos_top1': 0} +step=147500 lr=6.000000e-04 loss=2.2189 ce=2.2189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.053762727882713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020389489363878965, 'last_eos_top1': 0} +step=147550 lr=6.000000e-04 loss=3.0264 ce=3.0264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0539883987803478e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006755266804248095, 'last_eos_top1': 0} +step=147600 lr=6.000000e-04 loss=2.4721 ce=2.4721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.32938746700529e-07, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006933396798558533, 'last_eos_top1': 0} +step=147650 lr=6.000000e-04 loss=2.6403 ce=2.6403 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7269875317870174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011639641597867012, 'last_eos_top1': 0} +step=147700 lr=6.000000e-04 loss=2.6134 ce=2.6134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.330307092459407e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020255460403859615, 'last_eos_top1': 0} +step=147750 lr=6.000000e-04 loss=2.8397 ce=2.8397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.614565897209104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014330820413306355, 'last_eos_top1': 0} +step=147800 lr=6.000000e-04 loss=2.4444 ce=2.4444 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.500755833054427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008702484192326665, 'last_eos_top1': 0} +step=147850 lr=6.000000e-04 loss=3.1660 ce=3.1660 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7826764532364905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007018274627625942, 'last_eos_top1': 0} +step=147900 lr=6.000000e-04 loss=1.9905 ce=1.9905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3439256437995937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001026419922709465, 'last_eos_top1': 0} +step=147950 lr=6.000000e-04 loss=2.6733 ce=2.6733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2709245968144387e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007877185707911849, 'last_eos_top1': 0} +step=148000 lr=6.000000e-04 loss=1.9059 ce=1.9059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.376295014983043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010441953781992197, 'last_eos_top1': 0} +step=148050 lr=6.000000e-04 loss=2.8887 ce=2.8887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.934070486546261e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012188736582174897, 'last_eos_top1': 0} +step=148100 lr=6.000000e-04 loss=2.7259 ce=2.7259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.899247076333268e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013369955122470856, 'last_eos_top1': 0} +step=148150 lr=6.000000e-04 loss=2.5108 ce=2.5108 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8657145776378457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022899697069078684, 'last_eos_top1': 0} +step=148200 lr=6.000000e-04 loss=2.5496 ce=2.5496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.968569555785507e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002738203853368759, 'last_eos_top1': 0} +step=148250 lr=6.000000e-04 loss=2.7604 ce=2.7604 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0397121716086986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000936487689614296, 'last_eos_top1': 0} +step=148300 lr=6.000000e-04 loss=2.9991 ce=2.9991 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.983105671068188e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007124749827198684, 'last_eos_top1': 0} +step=148350 lr=6.000000e-04 loss=2.4318 ce=2.4318 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.856087576219579e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005372554296627641, 'last_eos_top1': 0} +step=148400 lr=6.000000e-04 loss=2.7392 ce=2.7392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7052803943661274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006824578158557415, 'last_eos_top1': 0} +step=148450 lr=6.000000e-04 loss=2.4343 ce=2.4343 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.350115887064021e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001023010932840407, 'last_eos_top1': 0} +step=148500 lr=6.000000e-04 loss=2.7169 ce=2.7169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6931166960130213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004174997447989881, 'last_eos_top1': 0} +step=148550 lr=6.000000e-04 loss=3.0536 ce=3.0536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.284440622839611e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006032891105860472, 'last_eos_top1': 0} +step=148600 lr=6.000000e-04 loss=2.0603 ce=2.0603 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0058720363304019e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005153340753167868, 'last_eos_top1': 0} +step=148650 lr=6.000000e-04 loss=2.4895 ce=2.4895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6141849452396855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005081335548311472, 'last_eos_top1': 0} +step=148700 lr=6.000000e-04 loss=2.5609 ce=2.5609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.099414688127581e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010076416656374931, 'last_eos_top1': 0} +step=148750 lr=6.000000e-04 loss=3.3835 ce=3.3835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0624879552051425e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009276921628043056, 'last_eos_top1': 0} +step=148800 lr=6.000000e-04 loss=2.6909 ce=2.6909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7026581998798065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013392651453614235, 'last_eos_top1': 0} +step=148850 lr=6.000000e-04 loss=2.6094 ce=2.6094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.308932592626661e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009886830812320113, 'last_eos_top1': 0} +step=148900 lr=6.000000e-04 loss=2.3204 ce=2.3204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.772487500304123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014366861432790756, 'last_eos_top1': 0} +step=148950 lr=6.000000e-04 loss=2.6641 ce=2.6641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.155042006459553e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019672675989568233, 'last_eos_top1': 0} +step=149000 lr=6.000000e-04 loss=2.5237 ce=2.5237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.070629984198604e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011617857962846756, 'last_eos_top1': 0} +step=149050 lr=6.000000e-04 loss=2.5510 ce=2.5510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.315827704383992e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020667086355388165, 'last_eos_top1': 0} +step=149100 lr=6.000000e-04 loss=2.1974 ce=2.1974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2986818546778522e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013640227261930704, 'last_eos_top1': 0} +step=149150 lr=6.000000e-04 loss=2.3072 ce=2.3072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.324282261019107e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019297772087156773, 'last_eos_top1': 0} +step=149200 lr=6.000000e-04 loss=2.5035 ce=2.5035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.23981452715816e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003136628307402134, 'last_eos_top1': 0} +step=149250 lr=6.000000e-04 loss=2.2604 ce=2.2604 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.039035280176904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012726247077807784, 'last_eos_top1': 0} +step=149300 lr=6.000000e-04 loss=2.6284 ce=2.6284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.433142410358414e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027795834466814995, 'last_eos_top1': 0} +step=149350 lr=6.000000e-04 loss=2.3641 ce=2.3641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.62756587594049e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003512715222314, 'last_eos_top1': 0} +step=149400 lr=6.000000e-04 loss=2.8436 ce=2.8436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.110606697504409e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034417707938700914, 'last_eos_top1': 0} +step=149450 lr=6.000000e-04 loss=1.9567 ce=1.9567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.621969426603755e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027756765484809875, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 + + +[elf_hfds direct rank=4 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2400000 rows=2391809 buffered=8191 part=openwebtext-t5 +step=149500 lr=6.000000e-04 loss=2.5364 ce=2.5364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.812015842820983e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00298277847468853, 'last_eos_top1': 0} +step=149550 lr=6.000000e-04 loss=2.4999 ce=2.4999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.987142453202978e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023243138566613197, 'last_eos_top1': 0} +step=149600 lr=6.000000e-04 loss=2.2389 ce=2.2389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.877790732076392e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017291036201640964, 'last_eos_top1': 0} +step=149650 lr=6.000000e-04 loss=2.4868 ce=2.4868 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.208548489259556e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003039639675989747, 'last_eos_top1': 0} +step=149700 lr=6.000000e-04 loss=2.5138 ce=2.5138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0924929887987673e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00263267382979393, 'last_eos_top1': 0} +step=149750 lr=6.000000e-04 loss=2.0716 ce=2.0716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.126065611373633e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003527708351612091, 'last_eos_top1': 0} +step=149800 lr=6.000000e-04 loss=2.4391 ce=2.4391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.510938874271233e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005046493373811245, 'last_eos_top1': 0} +step=149850 lr=6.000000e-04 loss=2.6670 ce=2.6670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.883781785902102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018641736824065447, 'last_eos_top1': 0} +step=149900 lr=6.000000e-04 loss=2.0766 ce=2.0766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1736332453438081e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002866607974283397, 'last_eos_top1': 0} +step=149950 lr=6.000000e-04 loss=2.3868 ce=2.3868 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8379620087216608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029207393527030945, 'last_eos_top1': 0} +step=150000 lr=6.000000e-04 loss=2.6926 ce=2.6926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4911141988704912e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005611836677417159, 'last_eos_top1': 0} +step=150050 lr=6.000000e-04 loss=2.1422 ce=2.1422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.228740519669373e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011428819270804524, 'last_eos_top1': 0} +step=150100 lr=6.000000e-04 loss=2.4116 ce=2.4116 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.420924031161121e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037105847150087357, 'last_eos_top1': 0} +step=150150 lr=6.000000e-04 loss=2.6740 ce=2.6740 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8107574406894855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031613081227988005, 'last_eos_top1': 0} +step=150200 lr=6.000000e-04 loss=3.0981 ce=3.0981 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.888370419095736e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003285755810793489, 'last_eos_top1': 0} +step=150250 lr=6.000000e-04 loss=2.8067 ce=2.8067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5455408376728883e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.290254244348034e-05, 'last_eos_top1': 0} +step=150300 lr=6.000000e-04 loss=2.1320 ce=2.1320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6834940146945883e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030928043997846544, 'last_eos_top1': 0} +step=150350 lr=6.000000e-04 loss=1.8143 ce=1.8143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.542325768852606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002521158312447369, 'last_eos_top1': 0} +step=150400 lr=6.000000e-04 loss=2.5311 ce=2.5311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.782577762962319e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00034229469019919634, 'last_eos_top1': 0} +step=150450 lr=6.000000e-04 loss=2.4525 ce=2.4525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.808004061691463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001335266511887312, 'last_eos_top1': 0} +step=150500 lr=6.000000e-04 loss=2.7067 ce=2.7067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.63376987783704e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020497239893302321, 'last_eos_top1': 0} +step=150550 lr=6.000000e-04 loss=2.3485 ce=2.3485 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.089974284695927e-05, 'pos0_bos_top1': 0, 'last_eos_p': 9.534064156468958e-05, 'last_eos_top1': 0} +step=150600 lr=6.000000e-04 loss=2.7125 ce=2.7125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.759854608797468e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.221657142508775e-05, 'last_eos_top1': 0} +step=150650 lr=6.000000e-04 loss=2.7568 ce=2.7568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.575907299411483e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022477158927358687, 'last_eos_top1': 0} +step=150700 lr=6.000000e-04 loss=2.2223 ce=2.2223 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.728222615492996e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006929590599611402, 'last_eos_top1': 0} +step=150750 lr=6.000000e-04 loss=2.6408 ce=2.6408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.192213499569334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016752441879361868, 'last_eos_top1': 0} +step=150800 lr=6.000000e-04 loss=2.3917 ce=2.3917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.928612618939951e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015750557649880648, 'last_eos_top1': 0} +step=150850 lr=6.000000e-04 loss=2.7369 ce=2.7369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8560262105893344e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026292232796549797, 'last_eos_top1': 0} +step=150900 lr=6.000000e-04 loss=2.6041 ce=2.6041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1679404451570008e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017845833208411932, 'last_eos_top1': 0} +step=150950 lr=6.000000e-04 loss=2.7238 ce=2.7238 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4535675290971994e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002734743058681488, 'last_eos_top1': 0} +step=151000 lr=6.000000e-04 loss=2.3213 ce=2.3213 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1999164598819334e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009488885989412665, 'last_eos_top1': 0} +step=151050 lr=6.000000e-04 loss=3.0327 ce=3.0327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1402091331547126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001257922500371933, 'last_eos_top1': 0} +step=151100 lr=6.000000e-04 loss=2.7052 ce=2.7052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0824642231455073e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011696993606165051, 'last_eos_top1': 0} +step=151150 lr=6.000000e-04 loss=2.4961 ce=2.4961 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.039844381826697e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007670402992516756, 'last_eos_top1': 0} +step=151200 lr=6.000000e-04 loss=2.6058 ce=2.6058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.624620709771989e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009986725635826588, 'last_eos_top1': 0} +step=151250 lr=6.000000e-04 loss=2.3718 ce=2.3718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.113444214861374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026328989770263433, 'last_eos_top1': 0} +step=151300 lr=6.000000e-04 loss=2.6888 ce=2.6888 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.222314404003555e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011777825420722365, 'last_eos_top1': 0} +step=151350 lr=6.000000e-04 loss=2.6034 ce=2.6034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.648421498946846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010738710407167673, 'last_eos_top1': 0} +step=151400 lr=6.000000e-04 loss=2.0598 ce=2.0598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.398482815304305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013840794563293457, 'last_eos_top1': 0} +step=151450 lr=6.000000e-04 loss=2.4940 ce=2.4940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.165617949387524e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019312987569719553, 'last_eos_top1': 0} +step=151500 lr=6.000000e-04 loss=1.9555 ce=1.9555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6182863115973305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002203651238232851, 'last_eos_top1': 0} +step=151550 lr=6.000000e-04 loss=2.3807 ce=2.3807 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.118604268413037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022894334979355335, 'last_eos_top1': 0} +step=151600 lr=6.000000e-04 loss=2.5826 ce=2.5826 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0264549928251654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003053526859730482, 'last_eos_top1': 0} +step=151650 lr=6.000000e-04 loss=2.4180 ce=2.4180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.144206963654142e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00376666197553277, 'last_eos_top1': 0} +step=151700 lr=6.000000e-04 loss=2.7284 ce=2.7284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.754106263746507e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002490060869604349, 'last_eos_top1': 0} +step=151750 lr=6.000000e-04 loss=2.3431 ce=2.3431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.286413043999346e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022237987723201513, 'last_eos_top1': 0} +step=151800 lr=6.000000e-04 loss=2.4569 ce=2.4569 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3492495781974867e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025672458577901125, 'last_eos_top1': 0} +step=151850 lr=6.000000e-04 loss=3.0714 ce=3.0714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.450875606678892e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001682417350821197, 'last_eos_top1': 0} +step=151900 lr=6.000000e-04 loss=2.3368 ce=2.3368 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.309292413992807e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002331925556063652, 'last_eos_top1': 0} +step=151950 lr=6.000000e-04 loss=2.5381 ce=2.5381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0569545338512398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013250040356069803, 'last_eos_top1': 0} +step=152000 lr=6.000000e-04 loss=2.3562 ce=2.3562 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.936116132128518e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018187868408858776, 'last_eos_top1': 0} +step=152050 lr=6.000000e-04 loss=2.4983 ce=2.4983 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.082692380005028e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001352568855509162, 'last_eos_top1': 0} +step=152100 lr=6.000000e-04 loss=2.4591 ce=2.4591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1644093067152426e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020164265297353268, 'last_eos_top1': 0} +step=152150 lr=6.000000e-04 loss=2.5492 ce=2.5492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.345574547594879e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018936670385301113, 'last_eos_top1': 0} +step=152200 lr=6.000000e-04 loss=2.0715 ce=2.0715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7834665413829498e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022273838985711336, 'last_eos_top1': 0} +step=152250 lr=6.000000e-04 loss=2.9701 ce=2.9701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3031223463476636e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015803712885826826, 'last_eos_top1': 0} +step=152300 lr=6.000000e-04 loss=2.3577 ce=2.3577 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.651768100913614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010139666264876723, 'last_eos_top1': 0} +step=152350 lr=6.000000e-04 loss=2.1710 ce=2.1710 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3152440412086435e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006191470893099904, 'last_eos_top1': 0} +step=152400 lr=6.000000e-04 loss=2.3432 ce=2.3432 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.892263733490836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007801283500157297, 'last_eos_top1': 0} +step=152450 lr=6.000000e-04 loss=3.1320 ce=3.1320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.858256140025333e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00141367525793612, 'last_eos_top1': 0} +step=152500 lr=6.000000e-04 loss=2.5400 ce=2.5400 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.404784355458105e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010499171912670135, 'last_eos_top1': 0} +step=152550 lr=6.000000e-04 loss=2.4611 ce=2.4611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.081995813176036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011675982968881726, 'last_eos_top1': 0} +step=152600 lr=6.000000e-04 loss=2.5731 ce=2.5731 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.899124855408445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008896907093003392, 'last_eos_top1': 0} +step=152650 lr=6.000000e-04 loss=2.3724 ce=2.3724 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.089392288937233e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008551840437576175, 'last_eos_top1': 0} +step=152700 lr=6.000000e-04 loss=1.9451 ce=1.9451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.537679498957004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011629101354628801, 'last_eos_top1': 0} +step=152750 lr=6.000000e-04 loss=2.0137 ce=2.0137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.890440893883351e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005688085802830756, 'last_eos_top1': 0} +step=152800 lr=6.000000e-04 loss=2.5810 ce=2.5810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4555102906306274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014950160402804613, 'last_eos_top1': 0} +step=152850 lr=6.000000e-04 loss=2.7311 ce=2.7311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.001548849279061e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002828887081705034, 'last_eos_top1': 0} +step=152900 lr=6.000000e-04 loss=2.2106 ce=2.2106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.200302555545932e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014449385344050825, 'last_eos_top1': 0} +step=152950 lr=6.000000e-04 loss=2.9452 ce=2.9452 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.79674770051497e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042461673729121685, 'last_eos_top1': 0} +step=153000 lr=6.000000e-04 loss=2.6780 ce=2.6780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3669284675852396e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000292169745080173, 'last_eos_top1': 0} +step=153050 lr=6.000000e-04 loss=2.7143 ce=2.7143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4595234360022005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023656387929804623, 'last_eos_top1': 0} +step=153100 lr=6.000000e-04 loss=2.3581 ce=2.3581 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3468552373960847e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031920141191221774, 'last_eos_top1': 0} +step=153150 lr=6.000000e-04 loss=2.7624 ce=2.7624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9558618734881748e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003350657061673701, 'last_eos_top1': 0} +step=153200 lr=6.000000e-04 loss=2.4866 ce=2.4866 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5775690371810924e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017229850345756859, 'last_eos_top1': 0} +step=153250 lr=6.000000e-04 loss=1.8434 ce=1.8434 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0623561946849804e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00068078120239079, 'last_eos_top1': 0} +step=153300 lr=6.000000e-04 loss=2.0482 ce=2.0482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.893759304039122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006866585463285446, 'last_eos_top1': 0} +step=153350 lr=6.000000e-04 loss=1.9432 ce=1.9432 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0890118978277314e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011082491255365312, 'last_eos_top1': 0} +step=153400 lr=6.000000e-04 loss=3.1445 ce=3.1445 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.578912012220826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023415905889123678, 'last_eos_top1': 0} +step=153450 lr=6.000000e-04 loss=3.1756 ce=3.1756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.942189894383773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.69784819567576e-05, 'last_eos_top1': 0} +step=153500 lr=6.000000e-04 loss=2.4777 ce=2.4777 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7922844765271293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.300707198330201e-05, 'last_eos_top1': 0} +step=153550 lr=6.000000e-04 loss=2.7244 ce=2.7244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.006151579640573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004005913797300309, 'last_eos_top1': 0} +step=153600 lr=6.000000e-04 loss=2.3251 ce=2.3251 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6121261877706274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007400850881822407, 'last_eos_top1': 0} +step=153650 lr=6.000000e-04 loss=2.8395 ce=2.8395 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.524209882423747e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007100780494511127, 'last_eos_top1': 0} +step=153700 lr=6.000000e-04 loss=2.6418 ce=2.6418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.485466888581868e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033996516140177846, 'last_eos_top1': 0} +step=153750 lr=6.000000e-04 loss=2.2809 ce=2.2809 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.933558663353324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010203116107732058, 'last_eos_top1': 0} +step=153800 lr=6.000000e-04 loss=2.3653 ce=2.3653 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3449828050797805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006402199505828321, 'last_eos_top1': 0} +step=153850 lr=6.000000e-04 loss=2.3197 ce=2.3197 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1778161933762021e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011352645233273506, 'last_eos_top1': 0} +step=153900 lr=6.000000e-04 loss=2.1992 ce=2.1992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.244614385446766e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000809657562058419, 'last_eos_top1': 0} +step=153950 lr=6.000000e-04 loss=1.9794 ce=1.9794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.153974704327993e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023142602294683456, 'last_eos_top1': 0} +step=154000 lr=6.000000e-04 loss=2.5531 ce=2.5531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2994149074074812e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022318768315017223, 'last_eos_top1': 0} +step=154050 lr=6.000000e-04 loss=2.3193 ce=2.3193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4844898942101281e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024200661573559046, 'last_eos_top1': 0} +step=154100 lr=6.000000e-04 loss=1.9388 ce=1.9388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0917008694377728e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012819780968129635, 'last_eos_top1': 0} +step=154150 lr=6.000000e-04 loss=2.5043 ce=2.5043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.074909207294695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013286927714943886, 'last_eos_top1': 0} +step=154200 lr=6.000000e-04 loss=2.7559 ce=2.7559 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.4487048803130165e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001011751824989915, 'last_eos_top1': 0} +step=154250 lr=6.000000e-04 loss=2.3448 ce=2.3448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.118392891745316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015060983132570982, 'last_eos_top1': 0} +step=154300 lr=6.000000e-04 loss=2.9656 ce=2.9656 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4590474165743217e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021595689468085766, 'last_eos_top1': 0} +step=154350 lr=6.000000e-04 loss=2.7178 ce=2.7178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.680798610934289e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001246349886059761, 'last_eos_top1': 0} +step=154400 lr=6.000000e-04 loss=2.5666 ce=2.5666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.995054718572646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009390802588313818, 'last_eos_top1': 0} +step=154450 lr=6.000000e-04 loss=2.2308 ce=2.2308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6029139159363694e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043068762170150876, 'last_eos_top1': 0} +step=154500 lr=6.000000e-04 loss=2.4193 ce=2.4193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5538858860963956e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007380243041552603, 'last_eos_top1': 0} +step=154550 lr=6.000000e-04 loss=2.9917 ce=2.9917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.300608897116035e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017522096168249846, 'last_eos_top1': 0} +step=154600 lr=6.000000e-04 loss=2.9378 ce=2.9378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.380167749535758e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002182330470532179, 'last_eos_top1': 0} +step=154650 lr=6.000000e-04 loss=2.4453 ce=2.4453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.85334554570727e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00281819817610085, 'last_eos_top1': 0} +step=154700 lr=6.000000e-04 loss=2.3317 ce=2.3317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.113622602133546e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025549973361194134, 'last_eos_top1': 0} +step=154750 lr=6.000000e-04 loss=2.2060 ce=2.2060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.22959827928571e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009808937320485711, 'last_eos_top1': 0} +step=154800 lr=6.000000e-04 loss=2.7499 ce=2.7499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7817067095602397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006902125896885991, 'last_eos_top1': 0} +step=154850 lr=6.000000e-04 loss=2.9490 ce=2.9490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4234111555852e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012689512223005295, 'last_eos_top1': 0} +step=154900 lr=6.000000e-04 loss=2.3794 ce=2.3794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5467760426399764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010881670750677586, 'last_eos_top1': 0} +step=154950 lr=6.000000e-04 loss=2.3493 ce=2.3493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.052503860904835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010519491042941809, 'last_eos_top1': 0} +step=155000 lr=6.000000e-04 loss=2.2374 ce=2.2374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.484924258780666e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006590880802832544, 'last_eos_top1': 0} +step=155050 lr=6.000000e-04 loss=2.4892 ce=2.4892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8223656702029984e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006109691457822919, 'last_eos_top1': 0} +step=155100 lr=6.000000e-04 loss=3.0738 ce=3.0738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.317049847988528e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002561570727266371, 'last_eos_top1': 0} +step=155150 lr=6.000000e-04 loss=2.2693 ce=2.2693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.972919330408331e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021122914040461183, 'last_eos_top1': 0} +step=155200 lr=6.000000e-04 loss=2.1992 ce=2.1992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.157468538323883e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004960477817803621, 'last_eos_top1': 0} +step=155250 lr=6.000000e-04 loss=2.8087 ce=2.8087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3791005737148225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006017748964950442, 'last_eos_top1': 0} +step=155300 lr=6.000000e-04 loss=2.3422 ce=2.3422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.285498910874594e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005133353406563401, 'last_eos_top1': 0} +step=155350 lr=6.000000e-04 loss=2.0694 ce=2.0694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.646788384998217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002231316757388413, 'last_eos_top1': 0} +step=155400 lr=6.000000e-04 loss=2.0219 ce=2.0219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.135642019624356e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012135277938796207, 'last_eos_top1': 0} +step=155450 lr=6.000000e-04 loss=2.8619 ce=2.8619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1591933798626997e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015044090105220675, 'last_eos_top1': 0} +step=155500 lr=6.000000e-04 loss=2.4148 ce=2.4148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3683503311767709e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042051024502143264, 'last_eos_top1': 0} +step=155550 lr=6.000000e-04 loss=2.6894 ce=2.6894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0377878425060771e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006429132772609591, 'last_eos_top1': 0} +step=155600 lr=6.000000e-04 loss=2.5008 ce=2.5008 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6505126596894115e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015304493717849255, 'last_eos_top1': 0} +step=155650 lr=6.000000e-04 loss=2.1352 ce=2.1352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7360744095640257e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044145641732029617, 'last_eos_top1': 0} +step=155700 lr=6.000000e-04 loss=2.6700 ce=2.6700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.835302989813499e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00034213988692499697, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2500000 rows=2491809 buffered=8191 part=openwebtext-t5 +step=155750 lr=6.000000e-04 loss=2.2439 ce=2.2439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4401013989700004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005503401625901461, 'last_eos_top1': 0} +step=155800 lr=6.000000e-04 loss=2.3958 ce=2.3958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.096366294310428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007430142723023891, 'last_eos_top1': 0} +step=155850 lr=6.000000e-04 loss=2.3999 ce=2.3999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.922066626837477e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007636617519892752, 'last_eos_top1': 0} +step=155900 lr=6.000000e-04 loss=2.1618 ce=2.1618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.396085831918754e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006411594222299755, 'last_eos_top1': 0} +step=155950 lr=6.000000e-04 loss=2.2496 ce=2.2496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.811679329781327e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000395027338527143, 'last_eos_top1': 0} +step=156000 lr=6.000000e-04 loss=2.2735 ce=2.2735 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.548603778355755e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005180794396437705, 'last_eos_top1': 0} +step=156050 lr=6.000000e-04 loss=2.0580 ce=2.0580 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3342778402147815e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005148493801243603, 'last_eos_top1': 0} +step=156100 lr=6.000000e-04 loss=2.4492 ce=2.4492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.511317562602926e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005453482735902071, 'last_eos_top1': 0} +step=156150 lr=6.000000e-04 loss=2.6226 ce=2.6226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.610793697182089e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044570211321115494, 'last_eos_top1': 0} +step=156200 lr=6.000000e-04 loss=2.4239 ce=2.4239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.301912016060669e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005588537314906716, 'last_eos_top1': 0} +step=156250 lr=6.000000e-04 loss=2.2083 ce=2.2083 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.322654033079743e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000514372019097209, 'last_eos_top1': 0} +step=156300 lr=6.000000e-04 loss=2.3287 ce=2.3287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.157040040008724e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008501417469233274, 'last_eos_top1': 0} +step=156350 lr=6.000000e-04 loss=2.9815 ce=2.9815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.773626642418094e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008768059778958559, 'last_eos_top1': 0} +step=156400 lr=6.000000e-04 loss=2.3775 ce=2.3775 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3209268217906356e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008774727466516197, 'last_eos_top1': 0} +step=156450 lr=6.000000e-04 loss=2.5441 ce=2.5441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.557772278232733e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013336179545149207, 'last_eos_top1': 0} +step=156500 lr=6.000000e-04 loss=2.4338 ce=2.4338 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5876776160439476e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001322530908510089, 'last_eos_top1': 0} +step=156550 lr=6.000000e-04 loss=2.1207 ce=2.1207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7357735487166792e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001703212154097855, 'last_eos_top1': 0} +step=156600 lr=6.000000e-04 loss=2.4780 ce=2.4780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0343022950110026e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013386739883571863, 'last_eos_top1': 0} +step=156650 lr=6.000000e-04 loss=2.1296 ce=2.1296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1957034075749107e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007738531567156315, 'last_eos_top1': 0} +step=156700 lr=6.000000e-04 loss=2.4304 ce=2.4304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.56843871588353e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005660379538312554, 'last_eos_top1': 0} +step=156750 lr=6.000000e-04 loss=2.3048 ce=2.3048 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7023372695111902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043458034633658826, 'last_eos_top1': 0} +step=156800 lr=6.000000e-04 loss=2.7674 ce=2.7674 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.129118214128539e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005098232068121433, 'last_eos_top1': 0} +step=156850 lr=6.000000e-04 loss=2.8176 ce=2.8176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.602105142723303e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004626183654181659, 'last_eos_top1': 0} +step=156900 lr=6.000000e-04 loss=2.9705 ce=2.9705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8721664421027526e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006929482333362103, 'last_eos_top1': 0} +step=156950 lr=6.000000e-04 loss=2.4230 ce=2.4230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.950170245545451e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006391038186848164, 'last_eos_top1': 0} +step=157000 lr=6.000000e-04 loss=2.1117 ce=2.1117 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.348276656353846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012629714328795671, 'last_eos_top1': 0} +step=157050 lr=6.000000e-04 loss=2.3351 ce=2.3351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.432165042089764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012236617039889097, 'last_eos_top1': 0} +step=157100 lr=6.000000e-04 loss=2.7899 ce=2.7899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.06817047810182e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010980560909956694, 'last_eos_top1': 0} +step=157150 lr=6.000000e-04 loss=2.3386 ce=2.3386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5893077943474054e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012697437778115273, 'last_eos_top1': 0} +step=157200 lr=6.000000e-04 loss=2.2192 ce=2.2192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.803261152934283e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008872980251908302, 'last_eos_top1': 0} +step=157250 lr=6.000000e-04 loss=2.6347 ce=2.6347 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2910450095660053e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009160427725873888, 'last_eos_top1': 0} +step=157300 lr=6.000000e-04 loss=2.3077 ce=2.3077 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2671715012402274e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007649693870916963, 'last_eos_top1': 0} +step=157350 lr=6.000000e-04 loss=2.3535 ce=2.3535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.327650822233409e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007897797040641308, 'last_eos_top1': 0} +step=157400 lr=6.000000e-04 loss=2.4641 ce=2.4641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.148810931947082e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006244269316084683, 'last_eos_top1': 0} +step=157450 lr=6.000000e-04 loss=2.4286 ce=2.4286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1031819667550735e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007254639058373868, 'last_eos_top1': 0} +step=157500 lr=6.000000e-04 loss=2.3703 ce=2.3703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.242785846348852e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044386726222001016, 'last_eos_top1': 0} +step=157550 lr=6.000000e-04 loss=2.9752 ce=2.9752 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.862692887603771e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026063379482366145, 'last_eos_top1': 0} +step=157600 lr=6.000000e-04 loss=2.2414 ce=2.2414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0979075492941774e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016703206347301602, 'last_eos_top1': 0} +step=157650 lr=6.000000e-04 loss=2.4228 ce=2.4228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2234906939265784e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019494802108965814, 'last_eos_top1': 0} +step=157700 lr=6.000000e-04 loss=2.4482 ce=2.4482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0185819923644885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021161629410926253, 'last_eos_top1': 0} +step=157750 lr=6.000000e-04 loss=2.7783 ce=2.7783 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.090268132335041e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005041296826675534, 'last_eos_top1': 0} +step=157800 lr=6.000000e-04 loss=2.7002 ce=2.7002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.102965133119142e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009108013473451138, 'last_eos_top1': 0} +step=157850 lr=6.000000e-04 loss=2.6729 ce=2.6729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.672533577831928e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005728768883273005, 'last_eos_top1': 0} +step=157900 lr=6.000000e-04 loss=2.7713 ce=2.7713 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.416128599084914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045436524669639766, 'last_eos_top1': 0} +step=157950 lr=6.000000e-04 loss=2.7356 ce=2.7356 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.947378329234198e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000494821579195559, 'last_eos_top1': 0} +step=158000 lr=6.000000e-04 loss=2.7046 ce=2.7046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.48595482239034e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000378887343686074, 'last_eos_top1': 0} +step=158050 lr=6.000000e-04 loss=3.0050 ce=3.0050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.154705599532463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021871352510061115, 'last_eos_top1': 0} +step=158100 lr=6.000000e-04 loss=2.5107 ce=2.5107 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.475548394111684e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021203540381975472, 'last_eos_top1': 0} +step=158150 lr=6.000000e-04 loss=2.4264 ce=2.4264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0362509783590212e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005231742979958653, 'last_eos_top1': 0} +step=158200 lr=6.000000e-04 loss=2.7516 ce=2.7516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.555673957744148e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003560387413017452, 'last_eos_top1': 0} +step=158250 lr=6.000000e-04 loss=2.4917 ce=2.4917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.338951945712324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042282347567379475, 'last_eos_top1': 0} +step=158300 lr=6.000000e-04 loss=2.7786 ce=2.7786 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8923814423033036e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006474175024777651, 'last_eos_top1': 0} +step=158350 lr=6.000000e-04 loss=2.5829 ce=2.5829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4254370398703031e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005470645846799016, 'last_eos_top1': 0} +step=158400 lr=6.000000e-04 loss=2.3176 ce=2.3176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2766153304255567e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008543698349967599, 'last_eos_top1': 0} +step=158450 lr=6.000000e-04 loss=2.6631 ce=2.6631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.576136572926771e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001290337648242712, 'last_eos_top1': 0} +step=158500 lr=6.000000e-04 loss=2.2087 ce=2.2087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4389494026545435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012427514884620905, 'last_eos_top1': 0} +step=158550 lr=6.000000e-04 loss=2.7904 ce=2.7904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8708050093846396e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015850544441491365, 'last_eos_top1': 0} +step=158600 lr=6.000000e-04 loss=2.3065 ce=2.3065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.561190583743155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013307032641023397, 'last_eos_top1': 0} +step=158650 lr=6.000000e-04 loss=2.9773 ce=2.9773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5610385162290186e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013543942477554083, 'last_eos_top1': 0} +step=158700 lr=6.000000e-04 loss=2.3812 ce=2.3812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.657858415972441e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019301529973745346, 'last_eos_top1': 0} +step=158750 lr=6.000000e-04 loss=2.4619 ce=2.4619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.295959560200572e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018866409081965685, 'last_eos_top1': 0} +step=158800 lr=6.000000e-04 loss=2.5549 ce=2.5549 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8042142983176745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002066275104880333, 'last_eos_top1': 0} +step=158850 lr=6.000000e-04 loss=3.1314 ce=3.1314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3282676920644008e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021720381919294596, 'last_eos_top1': 0} +step=158900 lr=6.000000e-04 loss=2.6520 ce=2.6520 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1835085388156585e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019218644592911005, 'last_eos_top1': 0} +step=158950 lr=6.000000e-04 loss=1.8920 ce=1.8920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.206309313303791e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020261970348656178, 'last_eos_top1': 0} +step=159000 lr=6.000000e-04 loss=2.7540 ce=2.7540 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.517785211850423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025125707034021616, 'last_eos_top1': 0} +step=159050 lr=6.000000e-04 loss=2.6438 ce=2.6438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.839654921961483e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002027401700615883, 'last_eos_top1': 0} +step=159100 lr=6.000000e-04 loss=2.6096 ce=2.6096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6864686232293025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017998787807300687, 'last_eos_top1': 0} +step=159150 lr=6.000000e-04 loss=2.7527 ce=2.7527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.678277476457879e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018807626329362392, 'last_eos_top1': 0} +step=159200 lr=6.000000e-04 loss=2.9136 ce=2.9136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.424181719921762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019294768571853638, 'last_eos_top1': 0} +step=159250 lr=6.000000e-04 loss=2.8442 ce=2.8442 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.881831384613179e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001238483702763915, 'last_eos_top1': 0} +step=159300 lr=6.000000e-04 loss=2.6468 ce=2.6468 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.994131242914591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019731256179511547, 'last_eos_top1': 0} +step=159350 lr=6.000000e-04 loss=3.0637 ce=3.0637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.678668574342737e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018772757612168789, 'last_eos_top1': 0} +step=159400 lr=6.000000e-04 loss=2.5844 ce=2.5844 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8783142877509817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008712945855222642, 'last_eos_top1': 0} +step=159450 lr=6.000000e-04 loss=2.6730 ce=2.6730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6401943867094815e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011887545697391033, 'last_eos_top1': 0} +step=159500 lr=6.000000e-04 loss=3.0111 ce=3.0111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0129791917279363e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001576591981574893, 'last_eos_top1': 0} +step=159550 lr=6.000000e-04 loss=2.3104 ce=2.3104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.053938078461215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019488590769469738, 'last_eos_top1': 0} +step=159600 lr=6.000000e-04 loss=2.5514 ce=2.5514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5032853298180271e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018143949564546347, 'last_eos_top1': 0} +step=159650 lr=6.000000e-04 loss=2.1677 ce=2.1677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0822326657944359e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019344454631209373, 'last_eos_top1': 0} +step=159700 lr=6.000000e-04 loss=2.8455 ce=2.8455 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4719908904226031e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001598588190972805, 'last_eos_top1': 0} +step=159750 lr=6.000000e-04 loss=2.6973 ce=2.6973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.087312042131089e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001159844920039177, 'last_eos_top1': 0} +step=159800 lr=6.000000e-04 loss=2.1749 ce=2.1749 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.805980542092584e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014944325666874647, 'last_eos_top1': 0} +step=159850 lr=6.000000e-04 loss=2.4667 ce=2.4667 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.099109951814171e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018307971768081188, 'last_eos_top1': 0} +step=159900 lr=6.000000e-04 loss=2.5379 ce=2.5379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.42988208407769e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001701192813925445, 'last_eos_top1': 0} +step=159950 lr=6.000000e-04 loss=3.0194 ce=3.0194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0266270692227408e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00209883414208889, 'last_eos_top1': 0} +step=160000 lr=6.000000e-04 loss=2.8535 ce=2.8535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.052897101151757e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018091332167387009, 'last_eos_top1': 0} +step=160050 lr=6.000000e-04 loss=2.3516 ce=2.3516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.515267265669536e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001988910138607025, 'last_eos_top1': 0} +step=160100 lr=6.000000e-04 loss=2.1851 ce=2.1851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.382464668306056e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017830153228715062, 'last_eos_top1': 0} +step=160150 lr=6.000000e-04 loss=2.2438 ce=2.2438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9183200492989272e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002250570571050048, 'last_eos_top1': 0} +step=160200 lr=6.000000e-04 loss=3.2059 ce=3.2059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0640625077940058e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013692410429939628, 'last_eos_top1': 0} +step=160250 lr=6.000000e-04 loss=1.9883 ce=1.9883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4241880307963584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016569304279983044, 'last_eos_top1': 0} +step=160300 lr=6.000000e-04 loss=2.4731 ce=2.4731 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.011462573951576e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016136275371536613, 'last_eos_top1': 0} +step=160350 lr=6.000000e-04 loss=2.4902 ce=2.4902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.341465218109079e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018916837871074677, 'last_eos_top1': 0} +step=160400 lr=6.000000e-04 loss=2.5605 ce=2.5605 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2352042176644318e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009773392230272293, 'last_eos_top1': 0} +step=160450 lr=6.000000e-04 loss=2.5960 ce=2.5960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1108324290253222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011650086380541325, 'last_eos_top1': 0} +step=160500 lr=6.000000e-04 loss=2.5027 ce=2.5027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.040596079197712e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009717929642647505, 'last_eos_top1': 0} +step=160550 lr=6.000000e-04 loss=2.2664 ce=2.2664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.397198154038051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014310635160654783, 'last_eos_top1': 0} +step=160600 lr=6.000000e-04 loss=2.9042 ce=2.9042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7390550460258964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036174734123051167, 'last_eos_top1': 0} +step=160650 lr=6.000000e-04 loss=2.6235 ce=2.6235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8718668545479886e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002206495264545083, 'last_eos_top1': 0} +step=160700 lr=6.000000e-04 loss=1.9439 ce=1.9439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3348146644129883e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005468029994517565, 'last_eos_top1': 0} +step=160750 lr=6.000000e-04 loss=2.5501 ce=2.5501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.023993713213713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012475010007619858, 'last_eos_top1': 0} +step=160800 lr=6.000000e-04 loss=2.9198 ce=2.9198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1525669833645225e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014155206736177206, 'last_eos_top1': 0} +step=160850 lr=6.000000e-04 loss=2.5730 ce=2.5730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6145309928106144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017002359963953495, 'last_eos_top1': 0} +step=160900 lr=6.000000e-04 loss=2.7029 ce=2.7029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0529114408418536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018943387549370527, 'last_eos_top1': 0} +step=160950 lr=6.000000e-04 loss=2.3033 ce=2.3033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.810767783666961e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018344488926231861, 'last_eos_top1': 0} +step=161000 lr=6.000000e-04 loss=2.1312 ce=2.1312 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.562375109642744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013877162709832191, 'last_eos_top1': 0} +step=161050 lr=6.000000e-04 loss=2.8947 ce=2.8947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.993153768533375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016932552680373192, 'last_eos_top1': 0} +step=161100 lr=6.000000e-04 loss=2.8047 ce=2.8047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0847134944924619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018111888784915209, 'last_eos_top1': 0} +step=161150 lr=6.000000e-04 loss=2.5598 ce=2.5598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.660951268917415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025067271199077368, 'last_eos_top1': 0} +step=161200 lr=6.000000e-04 loss=2.2665 ce=2.2665 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.352482235641219e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002705615246668458, 'last_eos_top1': 0} +step=161250 lr=6.000000e-04 loss=2.5101 ce=2.5101 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.968589438358322e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019703260622918606, 'last_eos_top1': 0} +step=161300 lr=6.000000e-04 loss=2.4933 ce=2.4933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2723361325915903e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020064921118319035, 'last_eos_top1': 0} +step=161350 lr=6.000000e-04 loss=2.6061 ce=2.6061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.557807864737697e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027661826461553574, 'last_eos_top1': 0} +step=161400 lr=6.000000e-04 loss=2.5677 ce=2.5677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7401943953009322e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003843635320663452, 'last_eos_top1': 0} +step=161450 lr=6.000000e-04 loss=2.4422 ce=2.4422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.158302080119029e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002192828804254532, 'last_eos_top1': 0} +step=161500 lr=6.000000e-04 loss=2.4176 ce=2.4176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0440571713843383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004140813834965229, 'last_eos_top1': 0} +step=161550 lr=6.000000e-04 loss=2.3782 ce=2.3782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1505558834178373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003712079022079706, 'last_eos_top1': 0} +step=161600 lr=6.000000e-04 loss=2.6540 ce=2.6540 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5719164366601035e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002939007245004177, 'last_eos_top1': 0} +step=161650 lr=6.000000e-04 loss=2.3323 ce=2.3323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.629240250622388e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025287740863859653, 'last_eos_top1': 0} +step=161700 lr=6.000000e-04 loss=2.4364 ce=2.4364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.065317312371917e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002405972220003605, 'last_eos_top1': 0} +step=161750 lr=6.000000e-04 loss=2.4706 ce=2.4706 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9007948139915243e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00188807665836066, 'last_eos_top1': 0} +step=161800 lr=6.000000e-04 loss=2.7337 ce=2.7337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1158428605995141e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023966138251125813, 'last_eos_top1': 0} +step=161850 lr=6.000000e-04 loss=2.3692 ce=2.3692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1458320841484237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011628327192738652, 'last_eos_top1': 0} +step=161900 lr=6.000000e-04 loss=2.2178 ce=2.2178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.39057213650085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001626266399398446, 'last_eos_top1': 0} +step=161950 lr=6.000000e-04 loss=2.5915 ce=2.5915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.054467459965963e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006318730302155018, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2600000 rows=2591809 buffered=8191 part=openwebtext-t5 +step=162000 lr=6.000000e-04 loss=1.7235 ce=1.7235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.631553904095199e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005853163311257958, 'last_eos_top1': 0} +step=162050 lr=6.000000e-04 loss=2.1262 ce=2.1262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.435888134437846e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016855697613209486, 'last_eos_top1': 0} +step=162100 lr=6.000000e-04 loss=2.6976 ce=2.6976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.875745955563616e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014655569102615118, 'last_eos_top1': 0} +step=162150 lr=6.000000e-04 loss=2.4050 ce=2.4050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.938525413919706e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001114342361688614, 'last_eos_top1': 0} +step=162200 lr=6.000000e-04 loss=2.7297 ce=2.7297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.015771759848576e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012379001127555966, 'last_eos_top1': 0} +step=162250 lr=6.000000e-04 loss=1.8678 ce=1.8678 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.849810349900508e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018926444463431835, 'last_eos_top1': 0} +step=162300 lr=6.000000e-04 loss=2.7142 ce=2.7142 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.759965567151085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016074677696451545, 'last_eos_top1': 0} +step=162350 lr=6.000000e-04 loss=2.7726 ce=2.7726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.534107463565306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008183508180081844, 'last_eos_top1': 0} +step=162400 lr=6.000000e-04 loss=2.7789 ce=2.7789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.551140465191565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016571285668760538, 'last_eos_top1': 0} +step=162450 lr=6.000000e-04 loss=2.6561 ce=2.6561 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.301283297536429e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001344042830169201, 'last_eos_top1': 0} +step=162500 lr=6.000000e-04 loss=2.5023 ce=2.5023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.445375326322392e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009495149133726954, 'last_eos_top1': 0} +step=162550 lr=6.000000e-04 loss=2.7397 ce=2.7397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.926905477186665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006290733581408858, 'last_eos_top1': 0} +step=162600 lr=6.000000e-04 loss=2.5125 ce=2.5125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0845074939425103e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020979014225304127, 'last_eos_top1': 0} +step=162650 lr=6.000000e-04 loss=2.2518 ce=2.2518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.523290150857065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030480646528303623, 'last_eos_top1': 0} +step=162700 lr=6.000000e-04 loss=2.1056 ce=2.1056 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8346772675577085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017902050167322159, 'last_eos_top1': 0} +step=162750 lr=6.000000e-04 loss=2.6188 ce=2.6188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0844205462490208e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022619334049522877, 'last_eos_top1': 0} +step=162800 lr=6.000000e-04 loss=2.7216 ce=2.7216 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8171019948786125e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002796652726829052, 'last_eos_top1': 0} +step=162850 lr=6.000000e-04 loss=2.8819 ce=2.8819 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.647526894696057e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018478194251656532, 'last_eos_top1': 0} +step=162900 lr=6.000000e-04 loss=2.8640 ce=2.8640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.125379276345484e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002685064449906349, 'last_eos_top1': 0} +step=162950 lr=6.000000e-04 loss=2.4035 ce=2.4035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.69679697137326e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003211383242160082, 'last_eos_top1': 0} +step=163000 lr=6.000000e-04 loss=2.8778 ce=2.8778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.561621037486475e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002982314443215728, 'last_eos_top1': 0} +step=163050 lr=6.000000e-04 loss=2.3954 ce=2.3954 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1521011401782744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026564733125269413, 'last_eos_top1': 0} +step=163100 lr=6.000000e-04 loss=2.5180 ce=2.5180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.456594746850897e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023198202252388, 'last_eos_top1': 0} +step=163150 lr=6.000000e-04 loss=2.5396 ce=2.5396 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.65635741117876e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014220455195754766, 'last_eos_top1': 0} +step=163200 lr=6.000000e-04 loss=2.3681 ce=2.3681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.65657581621781e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018589633982628584, 'last_eos_top1': 0} +step=163250 lr=6.000000e-04 loss=2.1299 ce=2.1299 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.305484511249233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002493418287485838, 'last_eos_top1': 0} +step=163300 lr=6.000000e-04 loss=1.9821 ce=1.9821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.974591426318511e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00229539442807436, 'last_eos_top1': 0} +step=163350 lr=6.000000e-04 loss=2.7200 ce=2.7200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.994635790149914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018327224534004927, 'last_eos_top1': 0} +step=163400 lr=6.000000e-04 loss=2.3678 ce=2.3678 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.289195203455165e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021861724089831114, 'last_eos_top1': 0} +step=163450 lr=6.000000e-04 loss=2.2883 ce=2.2883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.606183887924999e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026208278723061085, 'last_eos_top1': 0} +step=163500 lr=6.000000e-04 loss=2.3909 ce=2.3909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8997356973122805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032052253372967243, 'last_eos_top1': 0} +step=163550 lr=6.000000e-04 loss=3.1439 ce=3.1439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2309156772971619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023220721632242203, 'last_eos_top1': 0} +step=163600 lr=6.000000e-04 loss=2.8516 ce=2.8516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5254820027621463e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020823185332119465, 'last_eos_top1': 0} +step=163650 lr=6.000000e-04 loss=2.4535 ce=2.4535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3293869415065274e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017602527514100075, 'last_eos_top1': 0} +step=163700 lr=6.000000e-04 loss=2.6057 ce=2.6057 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.837001587380655e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001353041734546423, 'last_eos_top1': 0} +step=163750 lr=6.000000e-04 loss=2.7970 ce=2.7970 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0391160685685463e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013643063139170408, 'last_eos_top1': 0} +step=163800 lr=6.000000e-04 loss=2.5428 ce=2.5428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.127964297775179e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016840631142258644, 'last_eos_top1': 0} +step=163850 lr=6.000000e-04 loss=2.8560 ce=2.8560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9160498747369274e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022032535634934902, 'last_eos_top1': 0} +step=163900 lr=6.000000e-04 loss=2.4608 ce=2.4608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3505787137546577e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026398911140859127, 'last_eos_top1': 0} +step=163950 lr=6.000000e-04 loss=2.2061 ce=2.2061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1301595804980025e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013275445671752095, 'last_eos_top1': 0} +step=164000 lr=6.000000e-04 loss=2.1672 ce=2.1672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.601886293035932e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018213542643934488, 'last_eos_top1': 0} +step=164050 lr=6.000000e-04 loss=2.6965 ce=2.6965 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.995193765149452e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009651618311181664, 'last_eos_top1': 0} +step=164100 lr=6.000000e-04 loss=1.9128 ce=1.9128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1148974812822416e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016587665304541588, 'last_eos_top1': 0} +step=164150 lr=6.000000e-04 loss=2.1614 ce=2.1614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.067508704494685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017664741026237607, 'last_eos_top1': 0} +step=164200 lr=6.000000e-04 loss=3.1553 ce=3.1553 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.703338946565054e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001474291319027543, 'last_eos_top1': 0} +step=164250 lr=6.000000e-04 loss=2.9610 ce=2.9610 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.803638022276573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001598717411980033, 'last_eos_top1': 0} +step=164300 lr=6.000000e-04 loss=2.5429 ce=2.5429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7355927411699668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010091650765389204, 'last_eos_top1': 0} +step=164350 lr=6.000000e-04 loss=3.3607 ce=3.3607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6546924598515034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013786603230983019, 'last_eos_top1': 0} +step=164400 lr=6.000000e-04 loss=2.2063 ce=2.2063 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5784938770812005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016567970160394907, 'last_eos_top1': 0} +step=164450 lr=6.000000e-04 loss=2.4845 ce=2.4845 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.531773124152096e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012706132838502526, 'last_eos_top1': 0} +step=164500 lr=6.000000e-04 loss=2.4086 ce=2.4086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.521420800709166e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011758445762097836, 'last_eos_top1': 0} +step=164550 lr=6.000000e-04 loss=2.5353 ce=2.5353 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.468375431722961e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007650948828086257, 'last_eos_top1': 0} +step=164600 lr=6.000000e-04 loss=2.2619 ce=2.2619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.809230970044155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014169480418786407, 'last_eos_top1': 0} +step=164650 lr=6.000000e-04 loss=2.6280 ce=2.6280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.075575023103738e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001483672414906323, 'last_eos_top1': 0} +step=164700 lr=6.000000e-04 loss=2.3090 ce=2.3090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.349448540277081e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013645966537296772, 'last_eos_top1': 0} +step=164750 lr=6.000000e-04 loss=2.6267 ce=2.6267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.534546289709397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009377195965498686, 'last_eos_top1': 0} +step=164800 lr=6.000000e-04 loss=2.1945 ce=2.1945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.130292488000123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001082089962437749, 'last_eos_top1': 0} +step=164850 lr=6.000000e-04 loss=2.7912 ce=2.7912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.903660050738836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002381830709055066, 'last_eos_top1': 0} +step=164900 lr=6.000000e-04 loss=2.9448 ce=2.9448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.028753887221683e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001662451308220625, 'last_eos_top1': 0} +step=164950 lr=6.000000e-04 loss=2.5526 ce=2.5526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5477619399462128e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001364474301226437, 'last_eos_top1': 0} +step=165000 lr=6.000000e-04 loss=2.1920 ce=2.1920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.778269612870645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013414416462182999, 'last_eos_top1': 0} +step=165050 lr=6.000000e-04 loss=2.8617 ce=2.8617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.864560428861296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013773065293207765, 'last_eos_top1': 0} +step=165100 lr=6.000000e-04 loss=2.6021 ce=2.6021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.035041456518229e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001645750948227942, 'last_eos_top1': 0} +step=165150 lr=6.000000e-04 loss=2.3033 ce=2.3033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.989013526326744e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012855405220761895, 'last_eos_top1': 0} +step=165200 lr=6.000000e-04 loss=2.5531 ce=2.5531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.212227162905037e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002342464867979288, 'last_eos_top1': 0} +step=165250 lr=6.000000e-04 loss=2.6393 ce=2.6393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.527895609702682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002600932028144598, 'last_eos_top1': 0} +step=165300 lr=6.000000e-04 loss=3.1284 ce=3.1284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.589998752635438e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023754495196044445, 'last_eos_top1': 0} +step=165350 lr=6.000000e-04 loss=2.5879 ce=2.5879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.689952659944538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002877794671803713, 'last_eos_top1': 0} +step=165400 lr=6.000000e-04 loss=2.1282 ce=2.1282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.460062079189811e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023704576306045055, 'last_eos_top1': 0} +step=165450 lr=6.000000e-04 loss=2.4856 ce=2.4856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.414274149487028e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015672530280426145, 'last_eos_top1': 0} +step=165500 lr=6.000000e-04 loss=2.4195 ce=2.4195 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.860360604652669e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013607091968879104, 'last_eos_top1': 0} +step=165550 lr=6.000000e-04 loss=2.4024 ce=2.4024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.226641067361925e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001157656661234796, 'last_eos_top1': 0} +step=165600 lr=6.000000e-04 loss=2.2450 ce=2.2450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.310930984909646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010899961926043034, 'last_eos_top1': 0} +step=165650 lr=6.000000e-04 loss=2.8880 ce=2.8880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.888012416718993e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017059931997209787, 'last_eos_top1': 0} +step=165700 lr=6.000000e-04 loss=2.4176 ce=2.4176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.128503966669086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016594196204096079, 'last_eos_top1': 0} +step=165750 lr=6.000000e-04 loss=1.9692 ce=1.9692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.324842277332209e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014329355908557773, 'last_eos_top1': 0} +step=165800 lr=6.000000e-04 loss=2.4467 ce=2.4467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.109778147831094e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014921951806172729, 'last_eos_top1': 0} +step=165850 lr=6.000000e-04 loss=2.9671 ce=2.9671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5522075045737438e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002048519905656576, 'last_eos_top1': 0} +step=165900 lr=6.000000e-04 loss=2.2256 ce=2.2256 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.167344170331489e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012647847179323435, 'last_eos_top1': 0} +step=165950 lr=6.000000e-04 loss=2.3447 ce=2.3447 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.672607585438527e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011887148721143603, 'last_eos_top1': 0} +step=166000 lr=6.000000e-04 loss=2.3538 ce=2.3538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.911688852691441e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006470805965363979, 'last_eos_top1': 0} +step=166050 lr=6.000000e-04 loss=2.3810 ce=2.3810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.736136245104717e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011592521332204342, 'last_eos_top1': 0} +step=166100 lr=6.000000e-04 loss=2.6685 ce=2.6685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.646057499892777e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012903574388474226, 'last_eos_top1': 0} +step=166150 lr=6.000000e-04 loss=2.1854 ce=2.1854 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4345375651319046e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001133283949457109, 'last_eos_top1': 0} +step=166200 lr=6.000000e-04 loss=2.8129 ce=2.8129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.721535207063425e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021347790025174618, 'last_eos_top1': 0} +step=166250 lr=6.000000e-04 loss=2.5869 ce=2.5869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.513505816954421e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001689175609499216, 'last_eos_top1': 0} +step=166300 lr=6.000000e-04 loss=2.3064 ce=2.3064 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.299643817124888e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001042823540046811, 'last_eos_top1': 0} +step=166350 lr=6.000000e-04 loss=2.9851 ce=2.9851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.601284207543358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014179929858073592, 'last_eos_top1': 0} +step=166400 lr=6.000000e-04 loss=2.4860 ce=2.4860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.213059456669725e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009183145593851805, 'last_eos_top1': 0} +step=166450 lr=6.000000e-04 loss=2.2129 ce=2.2129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.246722816489637e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011713318526744843, 'last_eos_top1': 0} +step=166500 lr=6.000000e-04 loss=2.2536 ce=2.2536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.739284122479148e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001254822127521038, 'last_eos_top1': 0} +step=166550 lr=6.000000e-04 loss=3.0168 ce=3.0168 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3106885489833076e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001485210843384266, 'last_eos_top1': 0} +step=166600 lr=6.000000e-04 loss=2.3640 ce=2.3640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.822392838832457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012879313435405493, 'last_eos_top1': 0} +step=166650 lr=6.000000e-04 loss=2.3198 ce=2.3198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4563038348569535e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021091802045702934, 'last_eos_top1': 0} +step=166700 lr=6.000000e-04 loss=2.6462 ce=2.6462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7002066569111776e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002569529926404357, 'last_eos_top1': 0} +step=166750 lr=6.000000e-04 loss=3.0997 ce=3.0997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.649662282143254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022234374191612005, 'last_eos_top1': 0} +step=166800 lr=6.000000e-04 loss=2.1564 ce=2.1564 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1996964531135745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028042932972311974, 'last_eos_top1': 0} +step=166850 lr=6.000000e-04 loss=2.3820 ce=2.3820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.826135283219628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025450456887483597, 'last_eos_top1': 0} +step=166900 lr=6.000000e-04 loss=2.1269 ce=2.1269 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2416620847943705e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002713656984269619, 'last_eos_top1': 0} +step=166950 lr=6.000000e-04 loss=2.4459 ce=2.4459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.944429176452104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001488539157435298, 'last_eos_top1': 0} +step=167000 lr=6.000000e-04 loss=2.4664 ce=2.4664 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.555565621762071e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001016566064208746, 'last_eos_top1': 0} +step=167050 lr=6.000000e-04 loss=2.7275 ce=2.7275 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.994791536068078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015113912522792816, 'last_eos_top1': 0} +step=167100 lr=6.000000e-04 loss=2.2451 ce=2.2451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.895138540130574e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014545179437845945, 'last_eos_top1': 0} +step=167150 lr=6.000000e-04 loss=2.8417 ce=2.8417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.925437552738003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013800037559121847, 'last_eos_top1': 0} +step=167200 lr=6.000000e-04 loss=2.3490 ce=2.3490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7434904243127676e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017710649408400059, 'last_eos_top1': 0} +step=167250 lr=6.000000e-04 loss=2.6156 ce=2.6156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6416026483057067e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001383250579237938, 'last_eos_top1': 0} +step=167300 lr=6.000000e-04 loss=2.3989 ce=2.3989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.721780608932022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016669665928930044, 'last_eos_top1': 0} +step=167350 lr=6.000000e-04 loss=2.4246 ce=2.4246 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7570644558873028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017073964700102806, 'last_eos_top1': 0} +step=167400 lr=6.000000e-04 loss=2.6198 ce=2.6198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.456943760393187e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011943362187594175, 'last_eos_top1': 0} +step=167450 lr=6.000000e-04 loss=2.9987 ce=2.9987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1216463766177185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017235809937119484, 'last_eos_top1': 0} +step=167500 lr=6.000000e-04 loss=2.5505 ce=2.5505 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.162158686289331e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014582506846636534, 'last_eos_top1': 0} +step=167550 lr=6.000000e-04 loss=2.5225 ce=2.5225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1662841643556021e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013133703032508492, 'last_eos_top1': 0} +step=167600 lr=6.000000e-04 loss=2.3661 ce=2.3661 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.728101314161904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002548555377870798, 'last_eos_top1': 0} +step=167650 lr=6.000000e-04 loss=2.6335 ce=2.6335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.457269020960666e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018415098311379552, 'last_eos_top1': 0} +step=167700 lr=6.000000e-04 loss=2.5016 ce=2.5016 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9249608714017086e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015324329724535346, 'last_eos_top1': 0} +step=167750 lr=6.000000e-04 loss=2.3211 ce=2.3211 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.598172255209647e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013485571835190058, 'last_eos_top1': 0} +step=167800 lr=6.000000e-04 loss=2.3603 ce=2.3603 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.600136773864506e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012863322626799345, 'last_eos_top1': 0} +step=167850 lr=6.000000e-04 loss=2.4934 ce=2.4934 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.30201963961008e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001518739853054285, 'last_eos_top1': 0} +step=167900 lr=6.000000e-04 loss=2.9274 ce=2.9274 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6965500334626995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009242653613910079, 'last_eos_top1': 0} +step=167950 lr=6.000000e-04 loss=2.2078 ce=2.2078 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.609423285728553e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010912276338785887, 'last_eos_top1': 0} +step=168000 lr=6.000000e-04 loss=3.1496 ce=3.1496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.250883416214492e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010080939391627908, 'last_eos_top1': 0} +step=168050 lr=6.000000e-04 loss=2.3268 ce=2.3268 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.649036898423219e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000953370938077569, 'last_eos_top1': 0} +step=168100 lr=6.000000e-04 loss=2.7351 ce=2.7351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.702120789559558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013529934221878648, 'last_eos_top1': 0} +step=168150 lr=6.000000e-04 loss=2.8182 ce=2.8182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.409600013692398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011605536565184593, 'last_eos_top1': 0} +step=168200 lr=6.000000e-04 loss=2.3821 ce=2.3821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1550479030120187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015649074921384454, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2700000 rows=2691809 buffered=8191 part=openwebtext-t5 +step=168250 lr=6.000000e-04 loss=2.2221 ce=2.2221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.34235641680425e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013080730568617582, 'last_eos_top1': 0} +step=168300 lr=6.000000e-04 loss=2.6831 ce=2.6831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3008041100401897e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013271799543872476, 'last_eos_top1': 0} +step=168350 lr=6.000000e-04 loss=2.2616 ce=2.2616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.822224243776873e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013300387654453516, 'last_eos_top1': 0} +step=168400 lr=6.000000e-04 loss=2.6361 ce=2.6361 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.022547262953594e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010138879297301173, 'last_eos_top1': 0} +step=168450 lr=6.000000e-04 loss=2.4083 ce=2.4083 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.241247436060803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010170078603550792, 'last_eos_top1': 0} +step=168500 lr=6.000000e-04 loss=2.3640 ce=2.3640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.556987839052454e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011493824422359467, 'last_eos_top1': 0} +step=168550 lr=6.000000e-04 loss=2.3951 ce=2.3951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.414849046152085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001620323397219181, 'last_eos_top1': 0} +step=168600 lr=6.000000e-04 loss=2.6204 ce=2.6204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.093516948458273e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014225223567336798, 'last_eos_top1': 0} +step=168650 lr=6.000000e-04 loss=2.7944 ce=2.7944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.778779839398339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009929923107847571, 'last_eos_top1': 0} +step=168700 lr=6.000000e-04 loss=2.4839 ce=2.4839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.002305104222614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046792987268418074, 'last_eos_top1': 0} +step=168750 lr=6.000000e-04 loss=2.4821 ce=2.4821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.632113243336789e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005647316575050354, 'last_eos_top1': 0} +step=168800 lr=6.000000e-04 loss=2.5128 ce=2.5128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.57550730870571e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000600615341681987, 'last_eos_top1': 0} +step=168850 lr=6.000000e-04 loss=2.4672 ce=2.4672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.835055268718861e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037488050293177366, 'last_eos_top1': 0} +step=168900 lr=6.000000e-04 loss=2.6582 ce=2.6582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.561569428711664e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005177513230592012, 'last_eos_top1': 0} +step=168950 lr=6.000000e-04 loss=2.1893 ce=2.1893 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.948002242599614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005051294574514031, 'last_eos_top1': 0} +step=169000 lr=6.000000e-04 loss=2.5388 ce=2.5388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.637798600830138e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011000367812812328, 'last_eos_top1': 0} +step=169050 lr=6.000000e-04 loss=2.4459 ce=2.4459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.000816261395812e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006615801248699427, 'last_eos_top1': 0} +step=169100 lr=6.000000e-04 loss=2.5364 ce=2.5364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.118569904676406e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007424335344694555, 'last_eos_top1': 0} +step=169150 lr=6.000000e-04 loss=2.5755 ce=2.5755 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.64157574242563e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006635115714743733, 'last_eos_top1': 0} +step=169200 lr=6.000000e-04 loss=3.0747 ce=3.0747 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.116285824944498e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021818745881319046, 'last_eos_top1': 0} +step=169250 lr=6.000000e-04 loss=2.2454 ce=2.2454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.384538210113533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029355770675465465, 'last_eos_top1': 0} +step=169300 lr=6.000000e-04 loss=2.6501 ce=2.6501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5166827981593087e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011580297723412514, 'last_eos_top1': 0} +step=169350 lr=6.000000e-04 loss=2.3145 ce=2.3145 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1406867997720838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010022562928497791, 'last_eos_top1': 0} +step=169400 lr=6.000000e-04 loss=2.7133 ce=2.7133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.361177616374334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005257914890535176, 'last_eos_top1': 0} +step=169450 lr=6.000000e-04 loss=2.0320 ce=2.0320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5472255603963276e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007038147887215018, 'last_eos_top1': 0} +step=169500 lr=6.000000e-04 loss=2.4835 ce=2.4835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.961295290850103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004959310754202306, 'last_eos_top1': 0} +step=169550 lr=6.000000e-04 loss=2.2941 ce=2.2941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.65056029194966e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005863382248207927, 'last_eos_top1': 0} +step=169600 lr=6.000000e-04 loss=2.3260 ce=2.3260 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.159863111563027e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000546357361599803, 'last_eos_top1': 0} +step=169650 lr=6.000000e-04 loss=2.7188 ce=2.7188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.466713891015388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006560987094417214, 'last_eos_top1': 0} +step=169700 lr=6.000000e-04 loss=2.8701 ce=2.8701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.056727554299869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002592677192296833, 'last_eos_top1': 0} +step=169750 lr=6.000000e-04 loss=2.2834 ce=2.2834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8373649406130426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002502649440430105, 'last_eos_top1': 0} +step=169800 lr=6.000000e-04 loss=2.8908 ce=2.8908 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.205214903166052e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003250812296755612, 'last_eos_top1': 0} +step=169850 lr=6.000000e-04 loss=1.7133 ce=1.7133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0303964700142387e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003297979710623622, 'last_eos_top1': 0} +step=169900 lr=6.000000e-04 loss=2.5553 ce=2.5553 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0745759936980903e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004802052571903914, 'last_eos_top1': 0} +step=169950 lr=6.000000e-04 loss=2.4500 ce=2.4500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.142569979303516e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003143067006021738, 'last_eos_top1': 0} +step=170000 lr=6.000000e-04 loss=2.5837 ce=2.5837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2663512279686984e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002418974181637168, 'last_eos_top1': 0} +step=170050 lr=6.000000e-04 loss=2.1423 ce=2.1423 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5386911147506908e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031528540421277285, 'last_eos_top1': 0} +step=170100 lr=6.000000e-04 loss=2.4940 ce=2.4940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0507666956982575e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003837302210740745, 'last_eos_top1': 0} +step=170150 lr=6.000000e-04 loss=2.9556 ce=2.9556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.409172551182564e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003331782063469291, 'last_eos_top1': 0} +step=170200 lr=6.000000e-04 loss=2.4113 ce=2.4113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.966458492272068e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029290313250385225, 'last_eos_top1': 0} +step=170250 lr=6.000000e-04 loss=2.5754 ce=2.5754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.990417776047252e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002627610811032355, 'last_eos_top1': 0} +step=170300 lr=6.000000e-04 loss=2.6289 ce=2.6289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.007515473058447e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031597178895026445, 'last_eos_top1': 0} +step=170350 lr=6.000000e-04 loss=1.9729 ce=1.9729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2237989722052589e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015724814147688448, 'last_eos_top1': 0} +step=170400 lr=6.000000e-04 loss=2.6772 ce=2.6772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4779076233680826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014197229756973684, 'last_eos_top1': 0} +step=170450 lr=6.000000e-04 loss=3.0843 ce=3.0843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3048005712334998e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016536048497073352, 'last_eos_top1': 0} +step=170500 lr=6.000000e-04 loss=2.4867 ce=2.4867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0205721082456876e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018471047224011272, 'last_eos_top1': 0} +step=170550 lr=6.000000e-04 loss=2.3620 ce=2.3620 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.316038252407452e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015757919754832983, 'last_eos_top1': 0} +step=170600 lr=6.000000e-04 loss=2.3689 ce=2.3689 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.062968385667773e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002308656694367528, 'last_eos_top1': 0} +step=170650 lr=6.000000e-04 loss=2.9654 ce=2.9654 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.894736437359825e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.879261440597475e-05, 'last_eos_top1': 0} +step=170700 lr=6.000000e-04 loss=2.6391 ce=2.6391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3314449461176991e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018939297297038138, 'last_eos_top1': 0} +step=170750 lr=6.000000e-04 loss=2.2896 ce=2.2896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.790893505443819e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002877478254958987, 'last_eos_top1': 0} +step=170800 lr=6.000000e-04 loss=2.5627 ce=2.5627 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4413706089253537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020914834749419242, 'last_eos_top1': 0} +step=170850 lr=6.000000e-04 loss=2.6665 ce=2.6665 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.432546913652914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024138769367709756, 'last_eos_top1': 0} +step=170900 lr=6.000000e-04 loss=2.0727 ce=2.0727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1154264686629176e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026372933643870056, 'last_eos_top1': 0} +step=170950 lr=6.000000e-04 loss=2.2484 ce=2.2484 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.212596650177147e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003315228968858719, 'last_eos_top1': 0} +step=171000 lr=6.000000e-04 loss=2.5825 ce=2.5825 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0485450679407222e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018060665752273053, 'last_eos_top1': 0} +step=171050 lr=6.000000e-04 loss=1.7345 ce=1.7345 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.289482942316681e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012661315849982202, 'last_eos_top1': 0} +step=171100 lr=6.000000e-04 loss=2.5928 ce=2.5928 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.702058842871338e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021283500245772302, 'last_eos_top1': 0} +step=171150 lr=6.000000e-04 loss=2.7365 ce=2.7365 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3734093954553828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004349882365204394, 'last_eos_top1': 0} +step=171200 lr=6.000000e-04 loss=2.7130 ce=2.7130 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.194873038912192e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002737349714152515, 'last_eos_top1': 0} +step=171250 lr=6.000000e-04 loss=2.9830 ce=2.9830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.70017686591018e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003869893844239414, 'last_eos_top1': 0} +step=171300 lr=6.000000e-04 loss=2.7877 ce=2.7877 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1060673386964481e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006482796743512154, 'last_eos_top1': 0} +step=171350 lr=6.000000e-04 loss=2.5461 ce=2.5461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3710818974650465e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004138018121011555, 'last_eos_top1': 0} +step=171400 lr=6.000000e-04 loss=2.8327 ce=2.8327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9263794608123135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003219952341169119, 'last_eos_top1': 0} +step=171450 lr=6.000000e-04 loss=2.9433 ce=2.9433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7287744412897155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004489457351155579, 'last_eos_top1': 0} +step=171500 lr=6.000000e-04 loss=2.5044 ce=2.5044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.797379450697917e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046545008081011474, 'last_eos_top1': 0} +step=171550 lr=6.000000e-04 loss=3.0820 ce=3.0820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.466726285405457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003203547094017267, 'last_eos_top1': 0} +step=171600 lr=6.000000e-04 loss=2.4493 ce=2.4493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.1476682680659e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005270364927127957, 'last_eos_top1': 0} +step=171650 lr=6.000000e-04 loss=2.3541 ce=2.3541 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.007488653063774e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000516084604896605, 'last_eos_top1': 0} +step=171700 lr=6.000000e-04 loss=2.9010 ce=2.9010 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6607799554767553e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035579100949689746, 'last_eos_top1': 0} +step=171750 lr=6.000000e-04 loss=2.5722 ce=2.5722 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.885617717169225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005270469700917602, 'last_eos_top1': 0} +step=171800 lr=6.000000e-04 loss=2.7460 ce=2.7460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.735236744745634e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024821265833452344, 'last_eos_top1': 0} +step=171850 lr=6.000000e-04 loss=2.6864 ce=2.6864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.274985702882987e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032710214145481586, 'last_eos_top1': 0} +step=171900 lr=6.000000e-04 loss=2.1529 ce=2.1529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.576680112222675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030595771386288106, 'last_eos_top1': 0} +step=171950 lr=6.000000e-04 loss=2.7080 ce=2.7080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.923065252020024e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035121949622407556, 'last_eos_top1': 0} +step=172000 lr=6.000000e-04 loss=2.7380 ce=2.7380 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.870924259303138e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046536861918866634, 'last_eos_top1': 0} +step=172050 lr=6.000000e-04 loss=2.4373 ce=2.4373 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1218696272408124e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037070471444167197, 'last_eos_top1': 0} +step=172100 lr=6.000000e-04 loss=2.2384 ce=2.2384 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.283016202563886e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032246962655335665, 'last_eos_top1': 0} +step=172150 lr=6.000000e-04 loss=2.7004 ce=2.7004 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.047787686227821e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017732020933181047, 'last_eos_top1': 0} +step=172200 lr=6.000000e-04 loss=2.7122 ce=2.7122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.441915734787472e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019455354777164757, 'last_eos_top1': 0} +step=172250 lr=6.000000e-04 loss=2.7227 ce=2.7227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.306994120881427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019220917602069676, 'last_eos_top1': 0} +step=172300 lr=6.000000e-04 loss=1.9292 ce=1.9292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9112431952380575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016970105934888124, 'last_eos_top1': 0} +step=172350 lr=6.000000e-04 loss=2.2662 ce=2.2662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.374775239237351e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013900957128498703, 'last_eos_top1': 0} +step=172400 lr=6.000000e-04 loss=2.3395 ce=2.3395 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.097707905108109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035103881964460015, 'last_eos_top1': 0} +step=172450 lr=6.000000e-04 loss=2.2978 ce=2.2978 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.520520739490166e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023712319671176374, 'last_eos_top1': 0} +step=172500 lr=6.000000e-04 loss=2.7722 ce=2.7722 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2255402402370237e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001929051650222391, 'last_eos_top1': 0} +step=172550 lr=6.000000e-04 loss=2.9301 ce=2.9301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.715698312589666e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001539614167995751, 'last_eos_top1': 0} +step=172600 lr=6.000000e-04 loss=2.1914 ce=2.1914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.839191205974203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001479234779253602, 'last_eos_top1': 0} +step=172650 lr=6.000000e-04 loss=2.6542 ce=2.6542 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1185202260530787e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013016819139011204, 'last_eos_top1': 0} +step=172700 lr=6.000000e-04 loss=3.0244 ce=3.0244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.801112481800374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020551090710796416, 'last_eos_top1': 0} +step=172750 lr=6.000000e-04 loss=3.0962 ce=3.0962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1169583558512386e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001905874814838171, 'last_eos_top1': 0} +step=172800 lr=6.000000e-04 loss=2.4257 ce=2.4257 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0418924830446485e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027631170814856887, 'last_eos_top1': 0} +step=172850 lr=6.000000e-04 loss=2.8734 ce=2.8734 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0231728083454072e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018367735901847482, 'last_eos_top1': 0} +step=172900 lr=6.000000e-04 loss=2.7421 ce=2.7421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.56067163113039e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014169812493491918, 'last_eos_top1': 0} +step=172950 lr=6.000000e-04 loss=2.3916 ce=2.3916 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0066155482491013e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018202498904429376, 'last_eos_top1': 0} +step=173000 lr=6.000000e-04 loss=2.8892 ce=2.8892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.158893590210937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020112740457989275, 'last_eos_top1': 0} +step=173050 lr=6.000000e-04 loss=2.6446 ce=2.6446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5691981388954446e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033033546060323715, 'last_eos_top1': 0} +step=173100 lr=6.000000e-04 loss=2.6938 ce=2.6938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.871327165456023e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033846276346594095, 'last_eos_top1': 0} +step=173150 lr=6.000000e-04 loss=1.8023 ce=1.8023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.180655312462477e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016869325190782547, 'last_eos_top1': 0} +step=173200 lr=6.000000e-04 loss=2.3412 ce=2.3412 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0467404030787293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014933697821106762, 'last_eos_top1': 0} +step=173250 lr=6.000000e-04 loss=3.0031 ce=3.0031 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.56124428435578e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001225725281983614, 'last_eos_top1': 0} +step=173300 lr=6.000000e-04 loss=2.8451 ce=2.8451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3037522371159866e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013405224308371544, 'last_eos_top1': 0} +step=173350 lr=6.000000e-04 loss=2.4299 ce=2.4299 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.001590579922777e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019635689386632293, 'last_eos_top1': 0} +step=173400 lr=6.000000e-04 loss=2.2870 ce=2.2870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.25761038766359e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00021337972430046648, 'last_eos_top1': 0} +step=173450 lr=6.000000e-04 loss=2.8196 ce=2.8196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.668383659620304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003169830306433141, 'last_eos_top1': 0} +step=173500 lr=6.000000e-04 loss=2.4363 ce=2.4363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.429519322002307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015766904107294977, 'last_eos_top1': 0} +step=173550 lr=6.000000e-04 loss=2.7891 ce=2.7891 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9830305215436965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012188483378849924, 'last_eos_top1': 0} +step=173600 lr=6.000000e-04 loss=2.6371 ce=2.6371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5508165890641976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015588555834256113, 'last_eos_top1': 0} +step=173650 lr=6.000000e-04 loss=2.6620 ce=2.6620 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4193994906672742e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015081794117577374, 'last_eos_top1': 0} +step=173700 lr=6.000000e-04 loss=2.7872 ce=2.7872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6665287603245815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.484325371682644e-05, 'last_eos_top1': 0} +step=173750 lr=6.000000e-04 loss=2.6717 ce=2.6717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3330794647336006e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.173242582008243e-05, 'last_eos_top1': 0} +step=173800 lr=6.000000e-04 loss=2.6641 ce=2.6641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.794845608557807e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.064190089702606e-05, 'last_eos_top1': 0} +step=173850 lr=6.000000e-04 loss=2.7941 ce=2.7941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8619258450344205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011630891094682738, 'last_eos_top1': 0} +step=173900 lr=6.000000e-04 loss=2.3143 ce=2.3143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8744381122814957e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.786781887058169e-05, 'last_eos_top1': 0} +step=173950 lr=6.000000e-04 loss=2.6716 ce=2.6716 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.43271222441399e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.4820244763977826e-05, 'last_eos_top1': 0} +step=174000 lr=6.000000e-04 loss=2.5006 ce=2.5006 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.710676764967502e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.52008379297331e-05, 'last_eos_top1': 0} +step=174050 lr=6.000000e-04 loss=2.1124 ce=2.1124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.555848929361673e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.129307389026508e-05, 'last_eos_top1': 0} +step=174100 lr=6.000000e-04 loss=2.1059 ce=2.1059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.286704436002765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.710452751372941e-05, 'last_eos_top1': 0} +step=174150 lr=6.000000e-04 loss=2.2354 ce=2.2354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.30357158620609e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.796335244667716e-05, 'last_eos_top1': 0} +step=174200 lr=6.000000e-04 loss=2.2524 ce=2.2524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.992133315477986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.945272667100653e-05, 'last_eos_top1': 0} +step=174250 lr=6.000000e-04 loss=2.2155 ce=2.2155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0296250770334154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.2923674704507e-05, 'last_eos_top1': 0} +step=174300 lr=6.000000e-04 loss=2.5031 ce=2.5031 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.566573812146089e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.9637092160992324e-05, 'last_eos_top1': 0} +step=174350 lr=6.000000e-04 loss=2.9008 ce=2.9008 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4701294023543596e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.215472265263088e-05, 'last_eos_top1': 0} +step=174400 lr=6.000000e-04 loss=2.2844 ce=2.2844 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8882724361901637e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.43845237896312e-05, 'last_eos_top1': 0} +step=174450 lr=6.000000e-04 loss=2.3439 ce=2.3439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.161872650205623e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011034780618501827, 'last_eos_top1': 0} +[elf_hfds direct rank=4 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=2800000 rows=2791809 buffered=8191 part=openwebtext-t5 +step=174500 lr=6.000000e-04 loss=2.2199 ce=2.2199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.728713641568902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.8314715968444943e-05, 'last_eos_top1': 0} +step=174550 lr=6.000000e-04 loss=2.3988 ce=2.3988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.65787797010853e-07, 'pos0_bos_top1': 0, 'last_eos_p': 2.6104407879756764e-05, 'last_eos_top1': 0} +step=174600 lr=6.000000e-04 loss=2.2125 ce=2.2125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3547853541240329e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.4268607376143336e-05, 'last_eos_top1': 0} +step=174650 lr=6.000000e-04 loss=2.3503 ce=2.3503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2163861811131937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.3408867491525598e-05, 'last_eos_top1': 0} +step=174700 lr=6.000000e-04 loss=2.3682 ce=2.3682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8542784800956724e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.81304819509387e-05, 'last_eos_top1': 0} +step=174750 lr=6.000000e-04 loss=2.5020 ce=2.5020 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7200534330186201e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.551882011583075e-05, 'last_eos_top1': 0} +step=174800 lr=6.000000e-04 loss=2.2032 ce=2.2032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4549564134067623e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.77772326930426e-05, 'last_eos_top1': 0} +step=174850 lr=6.000000e-04 loss=2.2440 ce=2.2440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6256083149055485e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010690829367376864, 'last_eos_top1': 0} +step=174900 lr=6.000000e-04 loss=2.3728 ce=2.3728 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2688354874844663e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.124905535718426e-05, 'last_eos_top1': 0} +step=174950 lr=6.000000e-04 loss=2.4057 ce=2.4057 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.085601969767595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.155427010729909e-05, 'last_eos_top1': 0} +step=175000 lr=6.000000e-04 loss=2.3921 ce=2.3921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4116070562740788e-06, 'pos0_bos_top1': 0, 'last_eos_p': 2.8002647013636306e-05, 'last_eos_top1': 0} +step=175050 lr=6.000000e-04 loss=2.4484 ce=2.4484 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.411078068893403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.635435081785545e-05, 'last_eos_top1': 0} +step=175100 lr=6.000000e-04 loss=2.4654 ce=2.4654 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.210177965229377e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022018775052856654, 'last_eos_top1': 0} +step=175150 lr=6.000000e-04 loss=2.3464 ce=2.3464 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.263686150807189e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003107343800365925, 'last_eos_top1': 0} +step=175200 lr=6.000000e-04 loss=2.5292 ce=2.5292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.623218723689206e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042591625242494047, 'last_eos_top1': 0} +step=175250 lr=6.000000e-04 loss=2.3805 ce=2.3805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.998997388203861e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001884523662738502, 'last_eos_top1': 0} +step=175300 lr=6.000000e-04 loss=2.3799 ce=2.3799 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.818302164087072e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016545754624530673, 'last_eos_top1': 0} +step=175350 lr=6.000000e-04 loss=2.4712 ce=2.4712 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.355308192316443e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013928665430285037, 'last_eos_top1': 0} +step=175400 lr=6.000000e-04 loss=2.2622 ce=2.2622 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.191441919625504e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001018718903651461, 'last_eos_top1': 0} +step=175450 lr=6.000000e-04 loss=1.8833 ce=1.8833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.671950591728091e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014781401841901243, 'last_eos_top1': 0} +step=175500 lr=6.000000e-04 loss=2.6009 ce=2.6009 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9128468668204732e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.834593321196735e-05, 'last_eos_top1': 0} +step=175550 lr=6.000000e-04 loss=2.5860 ce=2.5860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8258619901607744e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013774019316770136, 'last_eos_top1': 0} +step=175600 lr=6.000000e-04 loss=2.6590 ce=2.6590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.296388048416702e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014576221292372793, 'last_eos_top1': 0} +step=175650 lr=6.000000e-04 loss=2.7453 ce=2.7453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.774107871227898e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00029754452407360077, 'last_eos_top1': 0} +step=175700 lr=6.000000e-04 loss=1.6065 ce=1.6065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8585176298511215e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.3442591605708e-05, 'last_eos_top1': 0} +step=175750 lr=6.000000e-04 loss=2.6080 ce=2.6080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.183835128264036e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011546636960702017, 'last_eos_top1': 0} +step=175800 lr=6.000000e-04 loss=1.9240 ce=1.9240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.77648414846044e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012990360846742988, 'last_eos_top1': 0} +step=175850 lr=6.000000e-04 loss=2.6121 ce=2.6121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.575728209805675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011412210005801171, 'last_eos_top1': 0} +step=175900 lr=6.000000e-04 loss=2.5470 ce=2.5470 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2133724339946639e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004195436486043036, 'last_eos_top1': 0} +step=175950 lr=6.000000e-04 loss=2.5429 ce=2.5429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.509505278198048e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033175438875332475, 'last_eos_top1': 0} +step=176000 lr=6.000000e-04 loss=2.1973 ce=2.1973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.27079111331841e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022146120318211615, 'last_eos_top1': 0} +step=176050 lr=6.000000e-04 loss=2.3637 ce=2.3637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.220509254082572e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011618612916208804, 'last_eos_top1': 0} +step=176100 lr=6.000000e-04 loss=2.2744 ce=2.2744 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1822753448504955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000273240206297487, 'last_eos_top1': 0} +step=176150 lr=6.000000e-04 loss=2.6262 ce=2.6262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.151737473672256e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014830110012553632, 'last_eos_top1': 0} +step=176200 lr=6.000000e-04 loss=2.5350 ce=2.5350 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7811101972474717e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.292374746408314e-05, 'last_eos_top1': 0} +step=176250 lr=6.000000e-04 loss=2.6165 ce=2.6165 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7709569369326346e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010703409498091787, 'last_eos_top1': 0} +step=176300 lr=6.000000e-04 loss=2.7073 ce=2.7073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8446543082245626e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002806567645166069, 'last_eos_top1': 0} +step=176350 lr=6.000000e-04 loss=2.5322 ce=2.5322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3696592254273128e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001793592527974397, 'last_eos_top1': 0} +step=176400 lr=6.000000e-04 loss=2.3586 ce=2.3586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2337873057695106e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002976717078126967, 'last_eos_top1': 0} +step=176450 lr=6.000000e-04 loss=2.7201 ce=2.7201 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.759833129559411e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024266992113552988, 'last_eos_top1': 0} +step=176500 lr=6.000000e-04 loss=2.8593 ce=2.8593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.772594704467338e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003115250146947801, 'last_eos_top1': 0} +step=176550 lr=6.000000e-04 loss=2.9412 ce=2.9412 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.00949419499375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003813315706793219, 'last_eos_top1': 0} +step=176600 lr=6.000000e-04 loss=2.7879 ce=2.7879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5447133086563554e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015846954192966223, 'last_eos_top1': 0} +step=176650 lr=6.000000e-04 loss=2.5425 ce=2.5425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8035049151876592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001067316479748115, 'last_eos_top1': 0} +step=176700 lr=6.000000e-04 loss=2.3661 ce=2.3661 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8633301124282298e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014277463196776807, 'last_eos_top1': 0} +step=176750 lr=6.000000e-04 loss=2.1233 ce=2.1233 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.086578312329948e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003066312347073108, 'last_eos_top1': 0} +step=176800 lr=6.000000e-04 loss=2.3333 ce=2.3333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.185852726048324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003821514255832881, 'last_eos_top1': 0} +step=176850 lr=6.000000e-04 loss=2.3497 ce=2.3497 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.556818683136953e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001135368729592301, 'last_eos_top1': 0} +step=176900 lr=6.000000e-04 loss=2.5686 ce=2.5686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7004025469068438e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020473095355555415, 'last_eos_top1': 0} +step=176950 lr=6.000000e-04 loss=2.6748 ce=2.6748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.543467305964441e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012109010276617482, 'last_eos_top1': 0} +step=177000 lr=6.000000e-04 loss=2.1135 ce=2.1135 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2674965950718615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016546057304367423, 'last_eos_top1': 0} +step=177050 lr=6.000000e-04 loss=2.6718 ce=2.6718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.466129550768528e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001840779441408813, 'last_eos_top1': 0} +step=177100 lr=6.000000e-04 loss=2.4353 ce=2.4353 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.242841896484606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011480523971840739, 'last_eos_top1': 0} +step=177150 lr=6.000000e-04 loss=2.2306 ce=2.2306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.556058567890432e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.992277824087068e-05, 'last_eos_top1': 0} +step=177200 lr=6.000000e-04 loss=2.5431 ce=2.5431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.178572559496388e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012073734251316637, 'last_eos_top1': 0} +step=177250 lr=6.000000e-04 loss=2.5748 ce=2.5748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6682616812177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.356819019652903e-05, 'last_eos_top1': 0} +step=177300 lr=6.000000e-04 loss=2.8945 ce=2.8945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0508407437300775e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.22896768618375e-05, 'last_eos_top1': 0} +step=177350 lr=6.000000e-04 loss=3.1984 ce=3.1984 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.219983904680703e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.92545945639722e-05, 'last_eos_top1': 0} +step=177400 lr=6.000000e-04 loss=2.4988 ce=2.4988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4507003217877354e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011099289986304939, 'last_eos_top1': 0} +step=177450 lr=6.000000e-04 loss=2.7753 ce=2.7753 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3923116714286152e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002437034563627094, 'last_eos_top1': 0} +step=177500 lr=6.000000e-04 loss=2.9362 ce=2.9362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9650290091231e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003168812836520374, 'last_eos_top1': 0} +step=177550 lr=6.000000e-04 loss=2.3609 ce=2.3609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.57762667388306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036841159453615546, 'last_eos_top1': 0} +step=177600 lr=6.000000e-04 loss=2.6837 ce=2.6837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3477628019463737e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002964201266877353, 'last_eos_top1': 0} +step=177650 lr=6.000000e-04 loss=2.4153 ce=2.4153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.354561264743097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017156251124106348, 'last_eos_top1': 0} +step=177700 lr=6.000000e-04 loss=2.5021 ce=2.5021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.030377112940187e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016178618534468114, 'last_eos_top1': 0} +step=177750 lr=6.000000e-04 loss=2.3635 ce=2.3635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.577730694814818e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013301693252287805, 'last_eos_top1': 0} +step=177800 lr=6.000000e-04 loss=2.3218 ce=2.3218 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.02724981540814e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002692205016501248, 'last_eos_top1': 0} +step=177850 lr=6.000000e-04 loss=2.8187 ce=2.8187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4430336199875455e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000270850898232311, 'last_eos_top1': 0} +step=177900 lr=6.000000e-04 loss=2.6098 ce=2.6098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.05645914725028e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024751329328864813, 'last_eos_top1': 0} +step=177950 lr=6.000000e-04 loss=2.6048 ce=2.6048 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.758470600383589e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001504786778241396, 'last_eos_top1': 0} +step=178000 lr=6.000000e-04 loss=2.1655 ce=2.1655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.276773430930916e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039385774289257824, 'last_eos_top1': 0} +step=178050 lr=6.000000e-04 loss=2.6851 ce=2.6851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.753676653810544e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002978978445753455, 'last_eos_top1': 0} +step=178100 lr=6.000000e-04 loss=2.1517 ce=2.1517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.846719886903884e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020531020709313452, 'last_eos_top1': 0} +step=178150 lr=6.000000e-04 loss=2.3371 ce=2.3371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1517669160384685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001990464807022363, 'last_eos_top1': 0} +step=178200 lr=6.000000e-04 loss=2.4803 ce=2.4803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.238101948634721e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001983974507311359, 'last_eos_top1': 0} +step=178250 lr=6.000000e-04 loss=2.9951 ce=2.9951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.679381734400522e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001248327607754618, 'last_eos_top1': 0} +step=178300 lr=6.000000e-04 loss=1.9521 ce=1.9521 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.101842129486613e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018708326388150454, 'last_eos_top1': 0} +step=178350 lr=6.000000e-04 loss=2.3317 ce=2.3317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.993082024564501e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001907380938064307, 'last_eos_top1': 0} +step=178400 lr=6.000000e-04 loss=2.7969 ce=2.7969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.453318408399355e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037897168658673763, 'last_eos_top1': 0} +step=178450 lr=6.000000e-04 loss=2.6644 ce=2.6644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.534517989261076e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044803315540775657, 'last_eos_top1': 0} +step=178500 lr=6.000000e-04 loss=2.8138 ce=2.8138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.489710358437151e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006263981340453029, 'last_eos_top1': 0} +step=178550 lr=6.000000e-04 loss=2.7704 ce=2.7704 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.463715635618428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006592078716494143, 'last_eos_top1': 0} +step=178600 lr=6.000000e-04 loss=2.1800 ce=2.1800 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.328295744926436e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005024423589929938, 'last_eos_top1': 0} +step=178650 lr=6.000000e-04 loss=2.2499 ce=2.2499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.001218818913912e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00034809636417776346, 'last_eos_top1': 0} +step=178700 lr=6.000000e-04 loss=2.7554 ce=2.7554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.63696153726778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014170832582749426, 'last_eos_top1': 0} +step=178750 lr=6.000000e-04 loss=2.3250 ce=2.3250 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1120551789645106e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023461348609998822, 'last_eos_top1': 0} +step=178800 lr=6.000000e-04 loss=2.6060 ce=2.6060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8104941520723514e-06, 'pos0_bos_top1': 0, 'last_eos_p': 3.052572355954908e-05, 'last_eos_top1': 0} +step=178850 lr=6.000000e-04 loss=2.1387 ce=2.1387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.222499228490051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.491942182648927e-05, 'last_eos_top1': 0} +step=178900 lr=6.000000e-04 loss=2.1998 ce=2.1998 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.167450116365217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010944163659587502, 'last_eos_top1': 0} +step=178950 lr=6.000000e-04 loss=2.2188 ce=2.2188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.448346771823708e-06, 'pos0_bos_top1': 0, 'last_eos_p': 7.553100294899195e-05, 'last_eos_top1': 0} +step=179000 lr=6.000000e-04 loss=2.3634 ce=2.3634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6115134207648225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.344440043903887e-05, 'last_eos_top1': 0} +step=179050 lr=6.000000e-04 loss=3.1225 ce=3.1225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.221145329414867e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016133359167724848, 'last_eos_top1': 0} +step=179100 lr=6.000000e-04 loss=2.2210 ce=2.2210 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.308882464305498e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001740902371238917, 'last_eos_top1': 0} +step=179150 lr=6.000000e-04 loss=2.9306 ce=2.9306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1569718430546345e-06, 'pos0_bos_top1': 0, 'last_eos_p': 9.034726826939732e-05, 'last_eos_top1': 0} +step=179200 lr=6.000000e-04 loss=2.3045 ce=2.3045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.712738129659556e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.212217365624383e-05, 'last_eos_top1': 0} +step=179250 lr=6.000000e-04 loss=2.3951 ce=2.3951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7751545935170725e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.6873225073795766e-05, 'last_eos_top1': 0} +step=179300 lr=6.000000e-04 loss=2.2961 ce=2.2961 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.32055958804267e-06, 'pos0_bos_top1': 0, 'last_eos_p': 5.201050953473896e-05, 'last_eos_top1': 0} +step=179350 lr=6.000000e-04 loss=3.0415 ce=3.0415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.680871486722026e-06, 'pos0_bos_top1': 0, 'last_eos_p': 4.6300665417220443e-05, 'last_eos_top1': 0} +step=179400 lr=6.000000e-04 loss=2.8091 ce=2.8091 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1363538255391177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.443079423159361e-05, 'last_eos_top1': 0} +step=179450 lr=6.000000e-04 loss=2.7124 ce=2.7124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1577230907278135e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000147903774632141, 'last_eos_top1': 0} +step=179500 lr=6.000000e-04 loss=2.4915 ce=2.4915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.272108294069767e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015598468598909676, 'last_eos_top1': 0} +step=179550 lr=6.000000e-04 loss=2.6885 ce=2.6885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.434300303226337e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016025151126086712, 'last_eos_top1': 0} +step=179600 lr=6.000000e-04 loss=2.4066 ce=2.4066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.829845470841974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010113560711033642, 'last_eos_top1': 0} +step=179650 lr=6.000000e-04 loss=2.5660 ce=2.5660 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.527762707264628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00014834405737929046, 'last_eos_top1': 0} +step=179700 lr=6.000000e-04 loss=2.7157 ce=2.7157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.651556527183857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002777073532342911, 'last_eos_top1': 0} +step=179750 lr=6.000000e-04 loss=2.4361 ce=2.4361 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.264203638129402e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046096922596916556, 'last_eos_top1': 0} +step=179800 lr=6.000000e-04 loss=2.8552 ce=2.8552 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.733509740617592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012860301649197936, 'last_eos_top1': 0} +step=179850 lr=6.000000e-04 loss=2.4203 ce=2.4203 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.832489245629404e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038934769690968096, 'last_eos_top1': 0} +step=179900 lr=6.000000e-04 loss=2.6013 ce=2.6013 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.56096950074425e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004290760844014585, 'last_eos_top1': 0} +step=179950 lr=6.000000e-04 loss=2.8551 ce=2.8551 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.064581728220219e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033826992148533463, 'last_eos_top1': 0} +step=180000 lr=6.000000e-04 loss=2.6740 ce=2.6740 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.704770046577323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030859775142744184, 'last_eos_top1': 0} +step=180050 lr=6.000000e-04 loss=2.3080 ce=2.3080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5022222214611247e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043741794070228934, 'last_eos_top1': 0} +step=180100 lr=6.000000e-04 loss=2.7548 ce=2.7548 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.831847142369952e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039682109490968287, 'last_eos_top1': 0} +step=180150 lr=6.000000e-04 loss=2.4712 ce=2.4712 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.431700290297158e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040711514884606004, 'last_eos_top1': 0} +step=180200 lr=6.000000e-04 loss=2.4911 ce=2.4911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.008670425217133e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037064525531604886, 'last_eos_top1': 0} +step=180250 lr=6.000000e-04 loss=2.5104 ce=2.5104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.937672434200067e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001886448881123215, 'last_eos_top1': 0} +step=180300 lr=6.000000e-04 loss=1.9642 ce=1.9642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.47157356372918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018672895384952426, 'last_eos_top1': 0} +step=180350 lr=6.000000e-04 loss=2.2838 ce=2.2838 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.698049335478572e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013618948287330568, 'last_eos_top1': 0} +step=180400 lr=6.000000e-04 loss=2.9304 ce=2.9304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.933715222752653e-06, 'pos0_bos_top1': 0, 'last_eos_p': 6.563649367308244e-05, 'last_eos_top1': 0} +step=180450 lr=6.000000e-04 loss=2.6215 ce=2.6215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.582056135404855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019259058171883225, 'last_eos_top1': 0} +step=180500 lr=6.000000e-04 loss=2.5306 ce=2.5306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.864628176117549e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023543008137494326, 'last_eos_top1': 0} +step=180550 lr=6.000000e-04 loss=2.8768 ce=2.8768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.529835339577403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010026318341260776, 'last_eos_top1': 0} +step=180600 lr=6.000000e-04 loss=2.2354 ce=2.2354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.608467861544341e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010766120249172673, 'last_eos_top1': 0} +step=180650 lr=6.000000e-04 loss=2.6989 ce=2.6989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.337386366794817e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016672132187522948, 'last_eos_top1': 0} +step=180700 lr=6.000000e-04 loss=2.4220 ce=2.4220 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8652102628257126e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017629633657634258, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=2900000 rows=2891809 buffered=8191 part=openwebtext-t5 +step=180750 lr=6.000000e-04 loss=2.7355 ce=2.7355 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.43043927589315e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001030007260851562, 'last_eos_top1': 0} +step=180800 lr=6.000000e-04 loss=2.2492 ce=2.2492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9195906538225245e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019346950284671038, 'last_eos_top1': 0} +step=180850 lr=6.000000e-04 loss=2.3271 ce=2.3271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.693771294725593e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019640948448795825, 'last_eos_top1': 0} +step=180900 lr=6.000000e-04 loss=2.7673 ce=2.7673 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.533374405786162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028225284768268466, 'last_eos_top1': 0} +step=180950 lr=6.000000e-04 loss=2.3637 ce=2.3637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.31230534761562e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002491887426003814, 'last_eos_top1': 0} +step=181000 lr=6.000000e-04 loss=2.5676 ce=2.5676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8504107326152734e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024111641687341034, 'last_eos_top1': 0} +step=181050 lr=6.000000e-04 loss=2.3958 ce=2.3958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.576171239838004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004767135833390057, 'last_eos_top1': 0} +step=181100 lr=6.000000e-04 loss=2.5498 ce=2.5498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1618394637480378e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004977025091648102, 'last_eos_top1': 0} +step=181150 lr=6.000000e-04 loss=2.1626 ce=2.1626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.949507911078399e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026625776081345975, 'last_eos_top1': 0} +step=181200 lr=6.000000e-04 loss=2.3663 ce=2.3663 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1371279217419215e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00010965040564769879, 'last_eos_top1': 0} +step=181250 lr=6.000000e-04 loss=2.8282 ce=2.8282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.894755304325372e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020831191795878112, 'last_eos_top1': 0} +step=181300 lr=6.000000e-04 loss=2.8394 ce=2.8394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.293352842272725e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020190543727949262, 'last_eos_top1': 0} +step=181350 lr=6.000000e-04 loss=2.8385 ce=2.8385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.434131485846592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018233153969049454, 'last_eos_top1': 0} +step=181400 lr=6.000000e-04 loss=2.2591 ce=2.2591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.166692633589264e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027233516448177397, 'last_eos_top1': 0} +step=181450 lr=6.000000e-04 loss=2.2911 ce=2.2911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9496515051287133e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.102334686554968e-05, 'last_eos_top1': 0} +step=181500 lr=6.000000e-04 loss=2.4503 ce=2.4503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6028656066046096e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015421214629895985, 'last_eos_top1': 0} +step=181550 lr=6.000000e-04 loss=2.3896 ce=2.3896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1101033022860065e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005076800007373095, 'last_eos_top1': 0} +step=181600 lr=6.000000e-04 loss=2.7947 ce=2.7947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.74120076635154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00040567581891082227, 'last_eos_top1': 0} +step=181650 lr=6.000000e-04 loss=1.9862 ce=1.9862 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.15852639687364e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046770775225013494, 'last_eos_top1': 0} +step=181700 lr=6.000000e-04 loss=2.6558 ce=2.6558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.814673153916374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011931152548640966, 'last_eos_top1': 0} +step=181750 lr=6.000000e-04 loss=2.5094 ce=2.5094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6967140911438037e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006521283648908138, 'last_eos_top1': 0} +step=181800 lr=6.000000e-04 loss=2.5676 ce=2.5676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1783034779000445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000538320979103446, 'last_eos_top1': 0} +step=181850 lr=6.000000e-04 loss=2.4781 ce=2.4781 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.314061698096339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006706904387101531, 'last_eos_top1': 0} +step=181900 lr=6.000000e-04 loss=2.5236 ce=2.5236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9934900592488702e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003408127522561699, 'last_eos_top1': 0} +step=181950 lr=6.000000e-04 loss=3.0690 ce=3.0690 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9100058276962955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028278009267523885, 'last_eos_top1': 0} +step=182000 lr=6.000000e-04 loss=3.1236 ce=3.1236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.623451902967645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002731244603637606, 'last_eos_top1': 0} +step=182050 lr=6.000000e-04 loss=2.5999 ce=2.5999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.918759936525021e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032354152062907815, 'last_eos_top1': 0} +step=182100 lr=6.000000e-04 loss=2.6391 ce=2.6391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0521187505219132e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042645499343052506, 'last_eos_top1': 0} +step=182150 lr=6.000000e-04 loss=2.8746 ce=2.8746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.418135621468537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007037037285044789, 'last_eos_top1': 0} +step=182200 lr=6.000000e-04 loss=2.7611 ce=2.7611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.318730134400539e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007853994611650705, 'last_eos_top1': 0} +step=182250 lr=6.000000e-04 loss=2.9320 ce=2.9320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.364857421838678e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009098321897909045, 'last_eos_top1': 0} +step=182300 lr=6.000000e-04 loss=2.5209 ce=2.5209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6080082281841896e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045421539107337594, 'last_eos_top1': 0} +step=182350 lr=6.000000e-04 loss=2.3986 ce=2.3986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.550854333094321e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017774917068891227, 'last_eos_top1': 0} +step=182400 lr=6.000000e-04 loss=2.5894 ce=2.5894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.727301191247534e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004970288136973977, 'last_eos_top1': 0} +step=182450 lr=6.000000e-04 loss=2.3247 ce=2.3247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4418360377894714e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006988170789554715, 'last_eos_top1': 0} +step=182500 lr=6.000000e-04 loss=2.2497 ce=2.2497 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.068032341019716e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010097299236804247, 'last_eos_top1': 0} +step=182550 lr=6.000000e-04 loss=2.2127 ce=2.2127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.523269126366358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006831286009401083, 'last_eos_top1': 0} +step=182600 lr=6.000000e-04 loss=2.7028 ce=2.7028 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.326384219486499e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045300202327780426, 'last_eos_top1': 0} +step=182650 lr=6.000000e-04 loss=2.1694 ce=2.1694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.076969839341473e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009314976632595062, 'last_eos_top1': 0} +step=182700 lr=6.000000e-04 loss=2.6591 ce=2.6591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.10028121425421e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005845521809533238, 'last_eos_top1': 0} +step=182750 lr=6.000000e-04 loss=2.7410 ce=2.7410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.317754246585537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011256968136876822, 'last_eos_top1': 0} +step=182800 lr=6.000000e-04 loss=2.2574 ce=2.2574 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2433385563781485e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020448784343898296, 'last_eos_top1': 0} +step=182850 lr=6.000000e-04 loss=2.9167 ce=2.9167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8675010475562885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001401948044076562, 'last_eos_top1': 0} +step=182900 lr=6.000000e-04 loss=2.5745 ce=2.5745 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.823704901151359e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001065682154148817, 'last_eos_top1': 0} +step=182950 lr=6.000000e-04 loss=2.9977 ce=2.9977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.158988981042057e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020752553828060627, 'last_eos_top1': 0} +step=183000 lr=6.000000e-04 loss=2.8249 ce=2.8249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.53313748241635e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014454765478149056, 'last_eos_top1': 0} +step=183050 lr=6.000000e-04 loss=2.6635 ce=2.6635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.015420356357936e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015285883564502, 'last_eos_top1': 0} +step=183100 lr=6.000000e-04 loss=2.4848 ce=2.4848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.443647751235403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001220498001202941, 'last_eos_top1': 0} +step=183150 lr=6.000000e-04 loss=2.3694 ce=2.3694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.217051693762187e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009147317614406347, 'last_eos_top1': 0} +step=183200 lr=6.000000e-04 loss=2.7197 ce=2.7197 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.786198703892296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005422953981906176, 'last_eos_top1': 0} +step=183250 lr=6.000000e-04 loss=2.1796 ce=2.1796 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.570011570758652e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003749938332475722, 'last_eos_top1': 0} +step=183300 lr=6.000000e-04 loss=2.5557 ce=2.5557 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3287751244206447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006769478786736727, 'last_eos_top1': 0} +step=183350 lr=6.000000e-04 loss=2.2024 ce=2.2024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.556274350557942e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043320556869730353, 'last_eos_top1': 0} +step=183400 lr=6.000000e-04 loss=2.2038 ce=2.2038 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.924269761540927e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000808167620562017, 'last_eos_top1': 0} +step=183450 lr=6.000000e-04 loss=2.4601 ce=2.4601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0043538168247323e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005425006384029984, 'last_eos_top1': 0} +step=183500 lr=6.000000e-04 loss=2.3132 ce=2.3132 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.684992280905135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004916285979561508, 'last_eos_top1': 0} +step=183550 lr=6.000000e-04 loss=2.5236 ce=2.5236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.100683923345059e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005412629107013345, 'last_eos_top1': 0} +step=183600 lr=6.000000e-04 loss=2.8914 ce=2.8914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.110374670242891e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00037541892379522324, 'last_eos_top1': 0} +step=183650 lr=6.000000e-04 loss=2.1272 ce=2.1272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.226273970038164e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007633133791387081, 'last_eos_top1': 0} +step=183700 lr=6.000000e-04 loss=2.8918 ce=2.8918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8551295397628564e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002676785516086966, 'last_eos_top1': 0} +step=183750 lr=6.000000e-04 loss=2.1345 ce=2.1345 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7106286729103886e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007317683193832636, 'last_eos_top1': 0} +step=183800 lr=6.000000e-04 loss=2.7545 ce=2.7545 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6732591221807525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00031091010896489024, 'last_eos_top1': 0} +step=183850 lr=6.000000e-04 loss=2.3359 ce=2.3359 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.72086367153679e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00045258572208695114, 'last_eos_top1': 0} +step=183900 lr=6.000000e-04 loss=2.6485 ce=2.6485 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.89574914961122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006564671057276428, 'last_eos_top1': 0} +step=183950 lr=6.000000e-04 loss=2.8763 ce=2.8763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5980116333812475e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001695820945315063, 'last_eos_top1': 0} +step=184000 lr=6.000000e-04 loss=3.0734 ce=3.0734 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.286435564106796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000984584679827094, 'last_eos_top1': 0} +step=184050 lr=6.000000e-04 loss=2.2709 ce=2.2709 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2518646144599188e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006701855454593897, 'last_eos_top1': 0} +step=184100 lr=6.000000e-04 loss=3.0626 ce=3.0626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.521198090718826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005537599790841341, 'last_eos_top1': 0} +step=184150 lr=6.000000e-04 loss=2.4128 ce=2.4128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.20286926580593e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030855584191158414, 'last_eos_top1': 0} +step=184200 lr=6.000000e-04 loss=2.4229 ce=2.4229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.121113720932044e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010582603281363845, 'last_eos_top1': 0} +step=184250 lr=6.000000e-04 loss=2.7412 ce=2.7412 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0070034477394074e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004655378870666027, 'last_eos_top1': 0} +step=184300 lr=6.000000e-04 loss=2.4016 ce=2.4016 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.592246907326626e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004913070588372648, 'last_eos_top1': 0} +step=184350 lr=6.000000e-04 loss=2.6163 ce=2.6163 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.147367002995452e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006398102850653231, 'last_eos_top1': 0} +step=184400 lr=6.000000e-04 loss=2.0756 ce=2.0756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0118137652170844e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006204941309988499, 'last_eos_top1': 0} +step=184450 lr=6.000000e-04 loss=2.6428 ce=2.6428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.686351414624369e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007065208628773689, 'last_eos_top1': 0} +step=184500 lr=6.000000e-04 loss=3.2335 ce=3.2335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.897486971662147e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009988886304199696, 'last_eos_top1': 0} +step=184550 lr=6.000000e-04 loss=2.0936 ce=2.0936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.65545405051671e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009277216158807278, 'last_eos_top1': 0} +step=184600 lr=6.000000e-04 loss=2.1029 ce=2.1029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5879060053266585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008041708497330546, 'last_eos_top1': 0} +step=184650 lr=6.000000e-04 loss=2.3886 ce=2.3886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4644074276002357e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010159271769225597, 'last_eos_top1': 0} +step=184700 lr=6.000000e-04 loss=2.4351 ce=2.4351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.413203674223041e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005386773264035583, 'last_eos_top1': 0} +step=184750 lr=6.000000e-04 loss=1.6833 ce=1.6833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.455342605069745e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044826450175605714, 'last_eos_top1': 0} +step=184800 lr=6.000000e-04 loss=2.3988 ce=2.3988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.702619207819225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007655194494873285, 'last_eos_top1': 0} +step=184850 lr=6.000000e-04 loss=2.0635 ce=2.0635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.036332029500045e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044063248788006604, 'last_eos_top1': 0} +step=184900 lr=6.000000e-04 loss=2.7310 ce=2.7310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.854491170495749e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007411773549392819, 'last_eos_top1': 0} +step=184950 lr=6.000000e-04 loss=2.3760 ce=2.3760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1359096535888966e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033707451075315475, 'last_eos_top1': 0} +step=185000 lr=6.000000e-04 loss=2.7531 ce=2.7531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.480994776822627e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004908801638521254, 'last_eos_top1': 0} +step=185050 lr=6.000000e-04 loss=2.0986 ce=2.0986 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4370357298030285e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004044789238832891, 'last_eos_top1': 0} +step=185100 lr=6.000000e-04 loss=2.8247 ce=2.8247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.791665898868814e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014581582508981228, 'last_eos_top1': 0} +step=185150 lr=6.000000e-04 loss=2.5253 ce=2.5253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.761570264439797e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012430476490408182, 'last_eos_top1': 0} +step=185200 lr=6.000000e-04 loss=3.0547 ce=3.0547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2291234219883336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009489371441304684, 'last_eos_top1': 0} +step=185250 lr=6.000000e-04 loss=2.2093 ce=2.2093 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.27416954759974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010625857394188643, 'last_eos_top1': 0} +step=185300 lr=6.000000e-04 loss=2.6045 ce=2.6045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.614555215695873e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007991766324266791, 'last_eos_top1': 0} +step=185350 lr=6.000000e-04 loss=3.0839 ce=3.0839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7862246244912967e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009344486752524972, 'last_eos_top1': 0} +step=185400 lr=6.000000e-04 loss=2.9314 ce=2.9314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.964647011249326e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046837993431836367, 'last_eos_top1': 0} +step=185450 lr=6.000000e-04 loss=2.2842 ce=2.2842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7294623830821365e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042897817911580205, 'last_eos_top1': 0} +step=185500 lr=6.000000e-04 loss=3.2103 ce=3.2103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.143242444522912e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005684025818482041, 'last_eos_top1': 0} +step=185550 lr=6.000000e-04 loss=3.1722 ce=3.1722 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.909790838311892e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007933067972771823, 'last_eos_top1': 0} +step=185600 lr=6.000000e-04 loss=2.7926 ce=2.7926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.529342615773203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024184450739994645, 'last_eos_top1': 0} +step=185650 lr=6.000000e-04 loss=2.9327 ce=2.9327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0038918844657019e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00018968860968016088, 'last_eos_top1': 0} +step=185700 lr=6.000000e-04 loss=2.3914 ce=2.3914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6228895876556635e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036536678089760244, 'last_eos_top1': 0} +step=185750 lr=6.000000e-04 loss=2.7855 ce=2.7855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.962045856402256e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044506974518299103, 'last_eos_top1': 0} +step=185800 lr=6.000000e-04 loss=2.5748 ce=2.5748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1894253475475125e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005926704034209251, 'last_eos_top1': 0} +step=185850 lr=6.000000e-04 loss=2.7504 ce=2.7504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3090368156554177e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036379252560436726, 'last_eos_top1': 0} +step=185900 lr=6.000000e-04 loss=2.3152 ce=2.3152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.991399797901977e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036734025343321264, 'last_eos_top1': 0} +step=185950 lr=6.000000e-04 loss=2.8034 ce=2.8034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.977915068593575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020726195361930877, 'last_eos_top1': 0} +step=186000 lr=6.000000e-04 loss=2.5509 ce=2.5509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.584288035403006e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001626199227757752, 'last_eos_top1': 0} +step=186050 lr=6.000000e-04 loss=2.8138 ce=2.8138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.422263934131479e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026314513524994254, 'last_eos_top1': 0} +step=186100 lr=6.000000e-04 loss=2.6509 ce=2.6509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.95367122918833e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00023532708291895688, 'last_eos_top1': 0} +step=186150 lr=6.000000e-04 loss=2.7545 ce=2.7545 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.494727702171076e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028062949422746897, 'last_eos_top1': 0} +step=186200 lr=6.000000e-04 loss=2.8902 ce=2.8902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8123844205983914e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019274948863312602, 'last_eos_top1': 0} +step=186250 lr=6.000000e-04 loss=2.4676 ce=2.4676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5837258565152297e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013752117229159921, 'last_eos_top1': 0} +step=186300 lr=6.000000e-04 loss=2.6866 ce=2.6866 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2536206617805874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 8.868683653417975e-05, 'last_eos_top1': 0} +step=186350 lr=6.000000e-04 loss=1.9072 ce=1.9072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6170040402794257e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003358513640705496, 'last_eos_top1': 0} +step=186400 lr=6.000000e-04 loss=2.5013 ce=2.5013 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.113570760935545e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007148798904381692, 'last_eos_top1': 0} +step=186450 lr=6.000000e-04 loss=2.8243 ce=2.8243 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8948750241397647e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00015756019274704158, 'last_eos_top1': 0} +step=186500 lr=6.000000e-04 loss=3.1044 ce=3.1044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.501816748525016e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00046731458860449493, 'last_eos_top1': 0} +step=186550 lr=6.000000e-04 loss=2.6286 ce=2.6286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.349958989128936e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006199396448209882, 'last_eos_top1': 0} +step=186600 lr=6.000000e-04 loss=2.3718 ce=2.3718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.429981451423373e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003625710087362677, 'last_eos_top1': 0} +step=186650 lr=6.000000e-04 loss=2.3984 ce=2.3984 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.453384801512584e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00020450641750358045, 'last_eos_top1': 0} +step=186700 lr=6.000000e-04 loss=2.3263 ce=2.3263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.55596273241099e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002072417119052261, 'last_eos_top1': 0} +step=186750 lr=6.000000e-04 loss=2.4500 ce=2.4500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.64172444480937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003569239634089172, 'last_eos_top1': 0} +step=186800 lr=6.000000e-04 loss=2.4393 ce=2.4393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1700622053467669e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033105292823165655, 'last_eos_top1': 0} +step=186850 lr=6.000000e-04 loss=2.0043 ce=2.0043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.247934834391344e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003036145935766399, 'last_eos_top1': 0} +step=186900 lr=6.000000e-04 loss=2.9793 ce=2.9793 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.907211743760854e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00032301226747222245, 'last_eos_top1': 0} +step=186950 lr=6.000000e-04 loss=2.5779 ce=2.5779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.121640242577996e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007136258063837886, 'last_eos_top1': 0} +[elf_hfds direct rank=3 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=2] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3000000 rows=2991809 buffered=8191 part=openwebtext-t5 +step=187000 lr=6.000000e-04 loss=2.9940 ce=2.9940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.643225555715617e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000649395165964961, 'last_eos_top1': 0} +step=187050 lr=6.000000e-04 loss=2.1789 ce=2.1789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4499823009828106e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007414026185870171, 'last_eos_top1': 0} +step=187100 lr=6.000000e-04 loss=2.8193 ce=2.8193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.504145752231125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003205481916666031, 'last_eos_top1': 0} +step=187150 lr=6.000000e-04 loss=2.5363 ce=2.5363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4719332688837312e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003964995266869664, 'last_eos_top1': 0} +step=187200 lr=6.000000e-04 loss=2.3892 ce=2.3892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.826541721238755e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005338292103260756, 'last_eos_top1': 0} +step=187250 lr=6.000000e-04 loss=2.8428 ce=2.8428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.938557246234268e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009555306751281023, 'last_eos_top1': 0} +step=187300 lr=6.000000e-04 loss=2.4919 ce=2.4919 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.74817750020884e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009667156264185905, 'last_eos_top1': 0} +step=187350 lr=6.000000e-04 loss=1.9951 ce=1.9951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.380640792893246e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000817180669400841, 'last_eos_top1': 0} +step=187400 lr=6.000000e-04 loss=2.4033 ce=2.4033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.857264452264644e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005149351200088859, 'last_eos_top1': 0} +step=187450 lr=6.000000e-04 loss=2.2952 ce=2.2952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.888691470725462e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005291757406666875, 'last_eos_top1': 0} +step=187500 lr=6.000000e-04 loss=2.4980 ce=2.4980 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.850245987763628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006328625604510307, 'last_eos_top1': 0} +step=187550 lr=6.000000e-04 loss=2.2015 ce=2.2015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.951305527356453e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005640535382553935, 'last_eos_top1': 0} +step=187600 lr=6.000000e-04 loss=2.4646 ce=2.4646 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.496509973250795e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007265418535098433, 'last_eos_top1': 0} +step=187650 lr=6.000000e-04 loss=2.5228 ce=2.5228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.170561624050606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006789491744711995, 'last_eos_top1': 0} +step=187700 lr=6.000000e-04 loss=3.1503 ce=3.1503 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.148655701399548e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008938491810113192, 'last_eos_top1': 0} +step=187750 lr=6.000000e-04 loss=2.4276 ce=2.4276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.010971214389428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007153183687478304, 'last_eos_top1': 0} +step=187800 lr=6.000000e-04 loss=2.3820 ce=2.3820 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9006766908423742e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009821371641010046, 'last_eos_top1': 0} +step=187850 lr=6.000000e-04 loss=2.4682 ce=2.4682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3466287732153432e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011154038365930319, 'last_eos_top1': 0} +step=187900 lr=6.000000e-04 loss=2.2717 ce=2.2717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.317530510888901e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008428820874541998, 'last_eos_top1': 0} +step=187950 lr=6.000000e-04 loss=2.2727 ce=2.2727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.057186000252841e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007690732600167394, 'last_eos_top1': 0} +step=188000 lr=6.000000e-04 loss=2.7119 ce=2.7119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.902106517576613e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008845917182043195, 'last_eos_top1': 0} +step=188050 lr=6.000000e-04 loss=2.2509 ce=2.2509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0791140893124975e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011719237081706524, 'last_eos_top1': 0} +step=188100 lr=6.000000e-04 loss=2.9206 ce=2.9206 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.267687578860205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009698715293779969, 'last_eos_top1': 0} +step=188150 lr=6.000000e-04 loss=2.9314 ce=2.9314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0137599929294083e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006511961692012846, 'last_eos_top1': 0} +step=188200 lr=6.000000e-04 loss=2.5001 ce=2.5001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.834462616709061e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005522762658074498, 'last_eos_top1': 0} +step=188250 lr=6.000000e-04 loss=3.0311 ce=3.0311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.939946848026011e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015216096071526408, 'last_eos_top1': 0} +step=188300 lr=6.000000e-04 loss=2.4541 ce=2.4541 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.195715392706916e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005646381177939475, 'last_eos_top1': 0} +step=188350 lr=6.000000e-04 loss=2.8612 ce=2.8612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.293692083796486e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00022422632900997996, 'last_eos_top1': 0} +step=188400 lr=6.000000e-04 loss=2.1288 ce=2.1288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8608087632164825e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002296418242622167, 'last_eos_top1': 0} +step=188450 lr=6.000000e-04 loss=2.3987 ce=2.3987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.740800694795325e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042894110083580017, 'last_eos_top1': 0} +step=188500 lr=6.000000e-04 loss=2.4318 ce=2.4318 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.035280992160551e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004926196415908635, 'last_eos_top1': 0} +step=188550 lr=6.000000e-04 loss=2.7410 ce=2.7410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.325667527562473e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044663704466074705, 'last_eos_top1': 0} +step=188600 lr=6.000000e-04 loss=1.9942 ce=1.9942 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.050556061789393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004634406650438905, 'last_eos_top1': 0} +step=188650 lr=6.000000e-04 loss=2.1518 ce=2.1518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1756839739973657e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007013482972979546, 'last_eos_top1': 0} +step=188700 lr=6.000000e-04 loss=2.4324 ce=2.4324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.103971827542409e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008229653467424214, 'last_eos_top1': 0} +step=188750 lr=6.000000e-04 loss=2.4996 ce=2.4996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.111378636182053e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014107488095760345, 'last_eos_top1': 0} +step=188800 lr=6.000000e-04 loss=2.3631 ce=2.3631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.847373929806054e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001050856662914157, 'last_eos_top1': 0} +step=188850 lr=6.000000e-04 loss=2.6861 ce=2.6861 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.754849669552641e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009800370316952467, 'last_eos_top1': 0} +step=188900 lr=6.000000e-04 loss=2.2021 ce=2.2021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.346574759343639e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004496329929679632, 'last_eos_top1': 0} +step=188950 lr=6.000000e-04 loss=2.8933 ce=2.8933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.123957973613869e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000587248126976192, 'last_eos_top1': 0} +step=189000 lr=6.000000e-04 loss=2.3132 ce=2.3132 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.122573828790337e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047548842849209905, 'last_eos_top1': 0} +step=189050 lr=6.000000e-04 loss=2.4783 ce=2.4783 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.89013426785823e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00030421296833083034, 'last_eos_top1': 0} +step=189100 lr=6.000000e-04 loss=2.4687 ce=2.4687 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4080162549798843e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00012259188224561512, 'last_eos_top1': 0} +step=189150 lr=6.000000e-04 loss=2.5849 ce=2.5849 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.898195584246423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00017125013982877135, 'last_eos_top1': 0} +step=189200 lr=6.000000e-04 loss=2.9384 ce=2.9384 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0702693543862551e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042094639502465725, 'last_eos_top1': 0} +step=189250 lr=6.000000e-04 loss=2.4257 ce=2.4257 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4093048927607015e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0001898174814414233, 'last_eos_top1': 0} +step=189300 lr=6.000000e-04 loss=2.4996 ce=2.4996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.18888747744495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003103303606621921, 'last_eos_top1': 0} +step=189350 lr=6.000000e-04 loss=2.2428 ce=2.2428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.651862127502682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004018995969090611, 'last_eos_top1': 0} +step=189400 lr=6.000000e-04 loss=2.1290 ce=2.1290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6385996786993928e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010422818595543504, 'last_eos_top1': 0} +step=189450 lr=6.000000e-04 loss=2.8047 ce=2.8047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.484818535071099e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003501887258607894, 'last_eos_top1': 0} +step=189500 lr=6.000000e-04 loss=2.4960 ce=2.4960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.535955541650765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006623214576393366, 'last_eos_top1': 0} +step=189550 lr=6.000000e-04 loss=2.7507 ce=2.7507 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3520037100533955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007413780549541116, 'last_eos_top1': 0} +step=189600 lr=6.000000e-04 loss=2.5794 ce=2.5794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.394826520292554e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008962866850197315, 'last_eos_top1': 0} +step=189650 lr=6.000000e-04 loss=2.4696 ce=2.4696 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2320349924266338e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005922851851209998, 'last_eos_top1': 0} +step=189700 lr=6.000000e-04 loss=2.0989 ce=2.0989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0224123798252549e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013343265745788813, 'last_eos_top1': 0} +step=189750 lr=6.000000e-04 loss=2.6216 ce=2.6216 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7806469259085134e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002029895782470703, 'last_eos_top1': 0} +step=189800 lr=6.000000e-04 loss=2.6560 ce=2.6560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0530007784836926e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002267375821247697, 'last_eos_top1': 0} +step=189850 lr=6.000000e-04 loss=2.4696 ce=2.4696 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.480710675939918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024457303807139397, 'last_eos_top1': 0} +step=189900 lr=6.000000e-04 loss=2.5683 ce=2.5683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3285571084416006e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011453831102699041, 'last_eos_top1': 0} +step=189950 lr=6.000000e-04 loss=2.6133 ce=2.6133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.772654286469333e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013372213579714298, 'last_eos_top1': 0} +step=190000 lr=6.000000e-04 loss=2.9920 ce=2.9920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.790248789911857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002269723918288946, 'last_eos_top1': 0} +step=190050 lr=6.000000e-04 loss=2.7306 ce=2.7306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.242522198997904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00204117177054286, 'last_eos_top1': 0} +step=190100 lr=6.000000e-04 loss=2.2093 ce=2.2093 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.502694304799661e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010784217156469822, 'last_eos_top1': 0} +step=190150 lr=6.000000e-04 loss=2.3918 ce=2.3918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9472507675527595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013461101334542036, 'last_eos_top1': 0} +step=190200 lr=6.000000e-04 loss=2.6524 ce=2.6524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.428453055676073e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014471892500296235, 'last_eos_top1': 0} +step=190250 lr=6.000000e-04 loss=2.1255 ce=2.1255 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2240130874706665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007345700869336724, 'last_eos_top1': 0} +step=190300 lr=6.000000e-04 loss=2.2354 ce=2.2354 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.317233338544611e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013363929465413094, 'last_eos_top1': 0} +step=190350 lr=6.000000e-04 loss=2.2179 ce=2.2179 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.044375368219335e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00117127550765872, 'last_eos_top1': 0} +step=190400 lr=6.000000e-04 loss=2.1628 ce=2.1628 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.907111234497279e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006166335660964251, 'last_eos_top1': 0} +step=190450 lr=6.000000e-04 loss=2.8180 ce=2.8180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.79503659234615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000500511028803885, 'last_eos_top1': 0} +step=190500 lr=6.000000e-04 loss=2.6381 ce=2.6381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.97164863999933e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006834609666839242, 'last_eos_top1': 0} +step=190550 lr=6.000000e-04 loss=2.6734 ce=2.6734 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.484216671902686e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002835447376128286, 'last_eos_top1': 0} +step=190600 lr=6.000000e-04 loss=2.5037 ce=2.5037 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.650191608059686e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004905376117676497, 'last_eos_top1': 0} +step=190650 lr=6.000000e-04 loss=2.7965 ce=2.7965 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1533637916727457e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00028223806293681264, 'last_eos_top1': 0} +step=190700 lr=6.000000e-04 loss=2.5776 ce=2.5776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.953361556341406e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002645542845129967, 'last_eos_top1': 0} +step=190750 lr=6.000000e-04 loss=2.5830 ce=2.5830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.008822881791275e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006284712580963969, 'last_eos_top1': 0} +step=190800 lr=6.000000e-04 loss=2.1768 ce=2.1768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.048695129517e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010305603500455618, 'last_eos_top1': 0} +step=190850 lr=6.000000e-04 loss=2.7641 ce=2.7641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.837103683210444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001183542888611555, 'last_eos_top1': 0} +step=190900 lr=6.000000e-04 loss=2.5792 ce=2.5792 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0936137186945416e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009342101402580738, 'last_eos_top1': 0} +step=190950 lr=6.000000e-04 loss=2.4346 ce=2.4346 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6415157006122172e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001043691299855709, 'last_eos_top1': 0} +step=191000 lr=6.000000e-04 loss=2.2323 ce=2.2323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2387506760424003e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012333919294178486, 'last_eos_top1': 0} +step=191050 lr=6.000000e-04 loss=2.3168 ce=2.3168 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4335334526549559e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009444914758205414, 'last_eos_top1': 0} +step=191100 lr=6.000000e-04 loss=2.7930 ce=2.7930 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7862486604135484e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013722128933295608, 'last_eos_top1': 0} +step=191150 lr=6.000000e-04 loss=1.6061 ce=1.6061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.478404738008976e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005504022701643407, 'last_eos_top1': 0} +step=191200 lr=6.000000e-04 loss=2.7880 ce=2.7880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1978132533840835e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004756727139465511, 'last_eos_top1': 0} +step=191250 lr=6.000000e-04 loss=2.1439 ce=2.1439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.294622719200561e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00033663175418041646, 'last_eos_top1': 0} +step=191300 lr=6.000000e-04 loss=2.4854 ce=2.4854 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.49035678684595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00042468501487746835, 'last_eos_top1': 0} +step=191350 lr=6.000000e-04 loss=2.4932 ce=2.4932 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5180151422100607e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010915629100054502, 'last_eos_top1': 0} +step=191400 lr=6.000000e-04 loss=2.1266 ce=2.1266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.217595910129603e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006234209868125618, 'last_eos_top1': 0} +step=191450 lr=6.000000e-04 loss=1.9494 ce=1.9494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.364872206177097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005062427371740341, 'last_eos_top1': 0} +step=191500 lr=6.000000e-04 loss=2.8316 ce=2.8316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8507989706413355e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005096272216178477, 'last_eos_top1': 0} +step=191550 lr=6.000000e-04 loss=2.7987 ce=2.7987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.719278750708327e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014151411596685648, 'last_eos_top1': 0} +step=191600 lr=6.000000e-04 loss=3.0181 ce=3.0181 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.242546194494935e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010063285008072853, 'last_eos_top1': 0} +step=191650 lr=6.000000e-04 loss=2.5277 ce=2.5277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.477949576335959e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006063904729671776, 'last_eos_top1': 0} +step=191700 lr=6.000000e-04 loss=2.5492 ce=2.5492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0210686923528556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015964690828695893, 'last_eos_top1': 0} +step=191750 lr=6.000000e-04 loss=2.5049 ce=2.5049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.429148586699739e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008893063059076667, 'last_eos_top1': 0} +step=191800 lr=6.000000e-04 loss=2.6462 ce=2.6462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9181010177126154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000746147648897022, 'last_eos_top1': 0} +step=191850 lr=6.000000e-04 loss=2.1046 ce=2.1046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.870529432490002e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004108530702069402, 'last_eos_top1': 0} +step=191900 lr=6.000000e-04 loss=2.6677 ce=2.6677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.40761310630478e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039310494321398437, 'last_eos_top1': 0} +step=191950 lr=6.000000e-04 loss=2.0543 ce=2.0543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0246265446767211e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003863434540107846, 'last_eos_top1': 0} +step=192000 lr=6.000000e-04 loss=2.2933 ce=2.2933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.151976883003954e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006575536681339145, 'last_eos_top1': 0} +step=192050 lr=6.000000e-04 loss=2.5392 ce=2.5392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.142811914993217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000657255994156003, 'last_eos_top1': 0} +step=192100 lr=6.000000e-04 loss=2.7182 ce=2.7182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1262365963775665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010813577100634575, 'last_eos_top1': 0} +step=192150 lr=6.000000e-04 loss=2.0314 ce=2.0314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.167879185639322e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008613607496954501, 'last_eos_top1': 0} +step=192200 lr=6.000000e-04 loss=2.4598 ce=2.4598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.415558452019468e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00079841457773, 'last_eos_top1': 0} +step=192250 lr=6.000000e-04 loss=1.9733 ce=1.9733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.356042599189095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007733844686299562, 'last_eos_top1': 0} +step=192300 lr=6.000000e-04 loss=2.3139 ce=2.3139 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.842098734225146e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006711413152515888, 'last_eos_top1': 0} +step=192350 lr=6.000000e-04 loss=2.3620 ce=2.3620 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.931646566954441e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008986148168332875, 'last_eos_top1': 0} +step=192400 lr=6.000000e-04 loss=2.2388 ce=2.2388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.755754839105066e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011889120796695352, 'last_eos_top1': 0} +step=192450 lr=6.000000e-04 loss=2.8223 ce=2.8223 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.326156497176271e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008194610709324479, 'last_eos_top1': 0} +step=192500 lr=6.000000e-04 loss=2.2474 ce=2.2474 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1601880032685585e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008087087189778686, 'last_eos_top1': 0} +step=192550 lr=6.000000e-04 loss=2.2874 ce=2.2874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1905178629094735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009215263416990638, 'last_eos_top1': 0} +step=192600 lr=6.000000e-04 loss=2.5611 ce=2.5611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.142037141718902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001180856255814433, 'last_eos_top1': 0} +step=192650 lr=6.000000e-04 loss=2.1762 ce=2.1762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.934004315757193e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002213668078184128, 'last_eos_top1': 0} +step=192700 lr=6.000000e-04 loss=2.2454 ce=2.2454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.24529285839526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018582554766908288, 'last_eos_top1': 0} +step=192750 lr=6.000000e-04 loss=2.4533 ce=2.4533 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.894900146406144e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015748234000056982, 'last_eos_top1': 0} +step=192800 lr=6.000000e-04 loss=2.6152 ce=2.6152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1200338526105043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009510671952739358, 'last_eos_top1': 0} +step=192850 lr=6.000000e-04 loss=2.6375 ce=2.6375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3183604096411727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001122607383877039, 'last_eos_top1': 0} +step=192900 lr=6.000000e-04 loss=2.4848 ce=2.4848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3863013009540737e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016749582719057798, 'last_eos_top1': 0} +step=192950 lr=6.000000e-04 loss=1.8968 ce=1.8968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.204134140512906e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017707431688904762, 'last_eos_top1': 0} +step=193000 lr=6.000000e-04 loss=2.4012 ce=2.4012 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.058570539200446e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009696896886453032, 'last_eos_top1': 0} +step=193050 lr=6.000000e-04 loss=2.7596 ce=2.7596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0655438574030995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008325148373842239, 'last_eos_top1': 0} +step=193100 lr=6.000000e-04 loss=2.1339 ce=2.1339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.458727456338238e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009940431918948889, 'last_eos_top1': 0} +step=193150 lr=6.000000e-04 loss=2.5224 ce=2.5224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.25281421787804e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012430973583832383, 'last_eos_top1': 0} +step=193200 lr=6.000000e-04 loss=2.0940 ce=2.0940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.312328074884135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007827345398254693, 'last_eos_top1': 0} +[elf_hfds direct rank=6 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3100000 rows=3091809 buffered=8191 part=openwebtext-t5 +step=193250 lr=6.000000e-04 loss=3.1212 ce=3.1212 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.590401426365133e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009234420722350478, 'last_eos_top1': 0} +step=193300 lr=6.000000e-04 loss=2.5850 ce=2.5850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3933406484720763e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014812437584623694, 'last_eos_top1': 0} +step=193350 lr=6.000000e-04 loss=2.6181 ce=2.6181 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.789225724583957e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016234652139246464, 'last_eos_top1': 0} +step=193400 lr=6.000000e-04 loss=2.2951 ce=2.2951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.665979107201565e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017728328239172697, 'last_eos_top1': 0} +step=193450 lr=6.000000e-04 loss=2.4329 ce=2.4329 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.83673761159298e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018398535903543234, 'last_eos_top1': 0} +step=193500 lr=6.000000e-04 loss=2.5811 ce=2.5811 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.592550340021262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001811453839763999, 'last_eos_top1': 0} +step=193550 lr=6.000000e-04 loss=2.4914 ce=2.4914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.699452463479247e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010700311977416277, 'last_eos_top1': 0} +step=193600 lr=6.000000e-04 loss=2.0821 ce=2.0821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0948495400953107e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009836999233812094, 'last_eos_top1': 0} +step=193650 lr=6.000000e-04 loss=2.4481 ce=2.4481 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.695931577065494e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000956315197981894, 'last_eos_top1': 0} +step=193700 lr=6.000000e-04 loss=2.6465 ce=2.6465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0835171855869703e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001243967330083251, 'last_eos_top1': 0} +step=193750 lr=6.000000e-04 loss=2.6243 ce=2.6243 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6248716241971124e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020203355234116316, 'last_eos_top1': 0} +step=193800 lr=6.000000e-04 loss=2.5310 ce=2.5310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7806231628346723e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012165360385552049, 'last_eos_top1': 0} +step=193850 lr=6.000000e-04 loss=1.9494 ce=1.9494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.769825070776278e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009195046150125563, 'last_eos_top1': 0} +step=193900 lr=6.000000e-04 loss=2.5940 ce=2.5940 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3149635833542561e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008534678490832448, 'last_eos_top1': 0} +step=193950 lr=6.000000e-04 loss=1.9540 ce=1.9540 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1940169315494131e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001182470005005598, 'last_eos_top1': 0} +step=194000 lr=6.000000e-04 loss=3.1478 ce=3.1478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.232039827911649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002004970796406269, 'last_eos_top1': 0} +step=194050 lr=6.000000e-04 loss=3.1942 ce=3.1942 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5852020775782876e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028931370470672846, 'last_eos_top1': 0} +step=194100 lr=6.000000e-04 loss=2.0682 ce=2.0682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.194347714772448e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025831637904047966, 'last_eos_top1': 0} +step=194150 lr=6.000000e-04 loss=2.2462 ce=2.2462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.330046067479998e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017093659844249487, 'last_eos_top1': 0} +step=194200 lr=6.000000e-04 loss=2.4159 ce=2.4159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5854119485302363e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002013238612562418, 'last_eos_top1': 0} +step=194250 lr=6.000000e-04 loss=2.0794 ce=2.0794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3632411486905767e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027295807376503944, 'last_eos_top1': 0} +step=194300 lr=6.000000e-04 loss=2.5644 ce=2.5644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.3436373238801025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001554744434542954, 'last_eos_top1': 0} +step=194350 lr=6.000000e-04 loss=2.5084 ce=2.5084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7742429918143898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002015945967286825, 'last_eos_top1': 0} +step=194400 lr=6.000000e-04 loss=2.7885 ce=2.7885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9425122445682064e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003489313181489706, 'last_eos_top1': 0} +step=194450 lr=6.000000e-04 loss=2.6056 ce=2.6056 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.607270057022106e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032199807465076447, 'last_eos_top1': 0} +step=194500 lr=6.000000e-04 loss=2.2578 ce=2.2578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.031108346680412e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002854161662980914, 'last_eos_top1': 0} +step=194550 lr=6.000000e-04 loss=2.9873 ce=2.9873 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3035592928645201e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029799141921103, 'last_eos_top1': 0} +step=194600 lr=6.000000e-04 loss=2.4049 ce=2.4049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.848300588695565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019412604160606861, 'last_eos_top1': 0} +step=194650 lr=6.000000e-04 loss=2.2097 ce=2.2097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.495620932080783e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014585816534236073, 'last_eos_top1': 0} +step=194700 lr=6.000000e-04 loss=2.4257 ce=2.4257 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4398070561583154e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002500429516658187, 'last_eos_top1': 0} +step=194750 lr=6.000000e-04 loss=1.9328 ce=1.9328 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.435430360492319e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003595977555960417, 'last_eos_top1': 0} +step=194800 lr=6.000000e-04 loss=2.8097 ce=2.8097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6862333723111078e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002943088300526142, 'last_eos_top1': 0} +step=194850 lr=6.000000e-04 loss=2.6592 ce=2.6592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3301115510985255e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013405253412202, 'last_eos_top1': 0} +step=194900 lr=6.000000e-04 loss=2.2959 ce=2.2959 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.630919647752307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001653196639381349, 'last_eos_top1': 0} +step=194950 lr=6.000000e-04 loss=2.7052 ce=2.7052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.751810022047721e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006043248577043414, 'last_eos_top1': 0} +step=195000 lr=6.000000e-04 loss=2.0484 ce=2.0484 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1436952566728e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00026906709535978734, 'last_eos_top1': 0} +step=195050 lr=6.000000e-04 loss=2.7607 ce=2.7607 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.555749107268639e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00016877707093954086, 'last_eos_top1': 0} +step=195100 lr=6.000000e-04 loss=2.4831 ce=2.4831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6951332731405273e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00011683860066113994, 'last_eos_top1': 0} +step=195150 lr=6.000000e-04 loss=2.1884 ce=2.1884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.701570782752242e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004788096121046692, 'last_eos_top1': 0} +step=195200 lr=6.000000e-04 loss=2.5491 ce=2.5491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.696624273492489e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00043206976260989904, 'last_eos_top1': 0} +step=195250 lr=6.000000e-04 loss=2.1414 ce=2.1414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.723292477137875e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003992850542999804, 'last_eos_top1': 0} +step=195300 lr=6.000000e-04 loss=2.9688 ce=2.9688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.944361641420983e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009039622964337468, 'last_eos_top1': 0} +step=195350 lr=6.000000e-04 loss=2.6675 ce=2.6675 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2108818737033289e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001317336456850171, 'last_eos_top1': 0} +step=195400 lr=6.000000e-04 loss=2.9280 ce=2.9280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7235182895092294e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008538550464436412, 'last_eos_top1': 0} +step=195450 lr=6.000000e-04 loss=2.2632 ce=2.2632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.640791646830621e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007202987908385694, 'last_eos_top1': 0} +step=195500 lr=6.000000e-04 loss=3.0852 ce=3.0852 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.393177732708864e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008474357309751213, 'last_eos_top1': 0} +step=195550 lr=6.000000e-04 loss=2.2144 ce=2.2144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.972325768903829e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008863441180437803, 'last_eos_top1': 0} +step=195600 lr=6.000000e-04 loss=2.7247 ce=2.7247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.000390617875382e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016299260314553976, 'last_eos_top1': 0} +step=195650 lr=6.000000e-04 loss=2.6418 ce=2.6418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.575330422085244e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001272543566301465, 'last_eos_top1': 0} +step=195700 lr=6.000000e-04 loss=3.2378 ce=3.2378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.434650812370819e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006442150915972888, 'last_eos_top1': 0} +step=195750 lr=6.000000e-04 loss=2.3905 ce=2.3905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.186762220750097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009619172778911889, 'last_eos_top1': 0} +step=195800 lr=6.000000e-04 loss=2.2847 ce=2.2847 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7091428971034475e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000674672017339617, 'last_eos_top1': 0} +step=195850 lr=6.000000e-04 loss=2.9324 ce=2.9324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.441992587089771e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003904086188413203, 'last_eos_top1': 0} +step=195900 lr=6.000000e-04 loss=2.6044 ce=2.6044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.854408638901077e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009172509890049696, 'last_eos_top1': 0} +step=195950 lr=6.000000e-04 loss=2.8100 ce=2.8100 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0145762644242495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000715624017175287, 'last_eos_top1': 0} +step=196000 lr=6.000000e-04 loss=2.1786 ce=2.1786 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9158650148892775e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005622656317427754, 'last_eos_top1': 0} +step=196050 lr=6.000000e-04 loss=2.6159 ce=2.6159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.294116934033809e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005615371046587825, 'last_eos_top1': 0} +step=196100 lr=6.000000e-04 loss=2.5842 ce=2.5842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0849220416275784e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007148862350732088, 'last_eos_top1': 0} +step=196150 lr=6.000000e-04 loss=2.8780 ce=2.8780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.306414318009047e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007713938248343766, 'last_eos_top1': 0} +step=196200 lr=6.000000e-04 loss=2.5084 ce=2.5084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.19946035335306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001095952233299613, 'last_eos_top1': 0} +step=196250 lr=6.000000e-04 loss=2.7084 ce=2.7084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5849789128405973e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015499498695135117, 'last_eos_top1': 0} +step=196300 lr=6.000000e-04 loss=2.7774 ce=2.7774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.937034307455178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012360342079773545, 'last_eos_top1': 0} +step=196350 lr=6.000000e-04 loss=2.6702 ce=2.6702 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1737510249076877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001375601626932621, 'last_eos_top1': 0} +step=196400 lr=6.000000e-04 loss=2.5094 ce=2.5094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.81867004459491e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001908737001940608, 'last_eos_top1': 0} +step=196450 lr=6.000000e-04 loss=2.8154 ce=2.8154 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.800256985821761e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021779430098831654, 'last_eos_top1': 0} +step=196500 lr=6.000000e-04 loss=2.9482 ce=2.9482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.19685462070629e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002797643421217799, 'last_eos_top1': 0} +step=196550 lr=6.000000e-04 loss=2.6641 ce=2.6641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.407468845281983e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013252352364361286, 'last_eos_top1': 0} +step=196600 lr=6.000000e-04 loss=2.3114 ce=2.3114 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7084174032788724e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021934579126536846, 'last_eos_top1': 0} +step=196650 lr=6.000000e-04 loss=2.5007 ce=2.5007 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2410911697079428e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035372204147279263, 'last_eos_top1': 0} +step=196700 lr=6.000000e-04 loss=2.7945 ce=2.7945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.570273060584441e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033365683630108833, 'last_eos_top1': 0} +step=196750 lr=6.000000e-04 loss=2.1524 ce=2.1524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.008601157082012e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002986451145261526, 'last_eos_top1': 0} +step=196800 lr=6.000000e-04 loss=2.6462 ce=2.6462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4698640370625071e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002257081214338541, 'last_eos_top1': 0} +step=196850 lr=6.000000e-04 loss=2.8537 ce=2.8537 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6021047485992312e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002464630641043186, 'last_eos_top1': 0} +step=196900 lr=6.000000e-04 loss=3.1848 ce=3.1848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.92086292069871e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022638430818915367, 'last_eos_top1': 0} +step=196950 lr=6.000000e-04 loss=2.5321 ce=2.5321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.790102583589032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022390868980437517, 'last_eos_top1': 0} +step=197000 lr=6.000000e-04 loss=2.2703 ce=2.2703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.968492693384178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032992339693009853, 'last_eos_top1': 0} +step=197050 lr=6.000000e-04 loss=2.0845 ce=2.0845 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2776003131875768e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003354673506692052, 'last_eos_top1': 0} +step=197100 lr=6.000000e-04 loss=2.7496 ce=2.7496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6972095181699842e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031445040367543697, 'last_eos_top1': 0} +step=197150 lr=6.000000e-04 loss=2.3863 ce=2.3863 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.411868409690214e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031858053989708424, 'last_eos_top1': 0} +step=197200 lr=6.000000e-04 loss=2.7613 ce=2.7613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.365300017321715e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002967449137941003, 'last_eos_top1': 0} +step=197250 lr=6.000000e-04 loss=2.8851 ce=2.8851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.522602340439335e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030829007737338543, 'last_eos_top1': 0} +step=197300 lr=6.000000e-04 loss=2.4907 ce=2.4907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.026708196150139e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003404526039958, 'last_eos_top1': 0} +step=197350 lr=6.000000e-04 loss=1.9528 ce=1.9528 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4701013242301997e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036212708801031113, 'last_eos_top1': 0} +step=197400 lr=6.000000e-04 loss=2.8451 ce=2.8451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.549014415213605e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004351228475570679, 'last_eos_top1': 0} +step=197450 lr=6.000000e-04 loss=2.0278 ce=2.0278 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.423059403459774e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003801831044256687, 'last_eos_top1': 0} +step=197500 lr=6.000000e-04 loss=2.4239 ce=2.4239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1462682272831444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035660285502672195, 'last_eos_top1': 0} +step=197550 lr=6.000000e-04 loss=2.1753 ce=2.1753 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.943110186810372e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031241835094988346, 'last_eos_top1': 0} +step=197600 lr=6.000000e-04 loss=2.5352 ce=2.5352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1336996976751834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002720148302614689, 'last_eos_top1': 0} +step=197650 lr=6.000000e-04 loss=2.6522 ce=2.6522 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1432104656705633e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029570171609520912, 'last_eos_top1': 0} +step=197700 lr=6.000000e-04 loss=2.2812 ce=2.2812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.37662890565116e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028794608078897, 'last_eos_top1': 0} +step=197750 lr=6.000000e-04 loss=2.5903 ce=2.5903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.55489078174287e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003162527922540903, 'last_eos_top1': 0} +step=197800 lr=6.000000e-04 loss=2.6926 ce=2.6926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.4707583078416064e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027504973113536835, 'last_eos_top1': 0} +step=197850 lr=6.000000e-04 loss=2.2513 ce=2.2513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3051710993750021e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00365086505189538, 'last_eos_top1': 0} +step=197900 lr=6.000000e-04 loss=2.7311 ce=2.7311 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3452003006241284e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002680849749594927, 'last_eos_top1': 0} +step=197950 lr=6.000000e-04 loss=2.6862 ce=2.6862 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.593569767370354e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020888959988951683, 'last_eos_top1': 0} +step=198000 lr=6.000000e-04 loss=2.4215 ce=2.4215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1242460459470749e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018280684016644955, 'last_eos_top1': 0} +step=198050 lr=6.000000e-04 loss=2.1539 ce=2.1539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.943463631614577e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018415616359561682, 'last_eos_top1': 0} +step=198100 lr=6.000000e-04 loss=2.1065 ce=2.1065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.749188964429777e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018598804017528892, 'last_eos_top1': 0} +step=198150 lr=6.000000e-04 loss=2.8052 ce=2.8052 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.822232090693433e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001736289239488542, 'last_eos_top1': 0} +step=198200 lr=6.000000e-04 loss=2.3111 ce=2.3111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.052159394836053e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010090398136526346, 'last_eos_top1': 0} +step=198250 lr=6.000000e-04 loss=2.2032 ce=2.2032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.641093023063149e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011932597262784839, 'last_eos_top1': 0} +step=198300 lr=6.000000e-04 loss=2.3352 ce=2.3352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.075058197689941e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011512110941112041, 'last_eos_top1': 0} +step=198350 lr=6.000000e-04 loss=2.1320 ce=2.1320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.532900210411754e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014948612079024315, 'last_eos_top1': 0} +step=198400 lr=6.000000e-04 loss=2.7539 ce=2.7539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.423302354756743e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001708245137706399, 'last_eos_top1': 0} +step=198450 lr=6.000000e-04 loss=3.1573 ce=3.1573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0661209317913745e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011478313244879246, 'last_eos_top1': 0} +step=198500 lr=6.000000e-04 loss=2.5015 ce=2.5015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.910819370707031e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009287184802815318, 'last_eos_top1': 0} +step=198550 lr=6.000000e-04 loss=2.3721 ce=2.3721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2572651939990465e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010331596713513136, 'last_eos_top1': 0} +step=198600 lr=6.000000e-04 loss=2.3926 ce=2.3926 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.810052127548261e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016035133739933372, 'last_eos_top1': 0} +step=198650 lr=6.000000e-04 loss=2.0977 ce=2.0977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.109616257774178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017276727594435215, 'last_eos_top1': 0} +step=198700 lr=6.000000e-04 loss=2.2876 ce=2.2876 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.000618522084551e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009766895091161132, 'last_eos_top1': 0} +step=198750 lr=6.000000e-04 loss=2.3616 ce=2.3616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1327880201861262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008169480133801699, 'last_eos_top1': 0} +step=198800 lr=6.000000e-04 loss=2.7188 ce=2.7188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.57084193942137e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005237638251855969, 'last_eos_top1': 0} +step=198850 lr=6.000000e-04 loss=2.7342 ce=2.7342 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.142146331607364e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007482438813894987, 'last_eos_top1': 0} +step=198900 lr=6.000000e-04 loss=2.6028 ce=2.6028 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1078263923991472e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006321121472865343, 'last_eos_top1': 0} +step=198950 lr=6.000000e-04 loss=2.4228 ce=2.4228 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2620688368042465e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006837885593995452, 'last_eos_top1': 0} +step=199000 lr=6.000000e-04 loss=1.9219 ce=1.9219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1582152839982882e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007034123409539461, 'last_eos_top1': 0} +step=199050 lr=6.000000e-04 loss=2.2617 ce=2.2617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5803121641511098e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006995530566200614, 'last_eos_top1': 0} +step=199100 lr=6.000000e-04 loss=2.8004 ce=2.8004 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.384549614158459e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008120601996779442, 'last_eos_top1': 0} +step=199150 lr=6.000000e-04 loss=2.3612 ce=2.3612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.60863133816747e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000892259122338146, 'last_eos_top1': 0} +step=199200 lr=6.000000e-04 loss=3.0411 ce=3.0411 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.49866137286881e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012334948405623436, 'last_eos_top1': 0} +step=199250 lr=6.000000e-04 loss=2.4947 ce=2.4947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7859005311038345e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011895564384758472, 'last_eos_top1': 0} +step=199300 lr=6.000000e-04 loss=2.8874 ce=2.8874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.720581718080211e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012220286298543215, 'last_eos_top1': 0} +step=199350 lr=6.000000e-04 loss=2.0692 ce=2.0692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2455662727006711e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012695686891674995, 'last_eos_top1': 0} +step=199400 lr=6.000000e-04 loss=2.1274 ce=2.1274 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0876154192374088e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010786522179841995, 'last_eos_top1': 0} +step=199450 lr=6.000000e-04 loss=2.4895 ce=2.4895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9711536879185587e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015997538575902581, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=4 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3200000 rows=3191809 buffered=8191 part=openwebtext-t5 +step=199500 lr=6.000000e-04 loss=1.9144 ce=1.9144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3930377463111654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015792913036420941, 'last_eos_top1': 0} +step=199550 lr=6.000000e-04 loss=2.3654 ce=2.3654 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3719001799472608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001600147457793355, 'last_eos_top1': 0} +step=199600 lr=6.000000e-04 loss=2.8673 ce=2.8673 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.738647349877283e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022202548570930958, 'last_eos_top1': 0} +step=199650 lr=6.000000e-04 loss=2.4301 ce=2.4301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.644029781455174e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001879200222902, 'last_eos_top1': 0} +step=199700 lr=6.000000e-04 loss=2.4099 ce=2.4099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5934421753627248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015821821289137006, 'last_eos_top1': 0} +step=199750 lr=6.000000e-04 loss=2.5375 ce=2.5375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.442035323241726e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014505801955237985, 'last_eos_top1': 0} +step=199800 lr=6.000000e-04 loss=2.5765 ce=2.5765 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.238195171841653e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016728148330003023, 'last_eos_top1': 0} +step=199850 lr=6.000000e-04 loss=2.7668 ce=2.7668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.684623036155244e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002135602757334709, 'last_eos_top1': 0} +step=199900 lr=6.000000e-04 loss=2.4132 ce=2.4132 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8453407392371446e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00241103395819664, 'last_eos_top1': 0} +step=199950 lr=6.000000e-04 loss=2.0833 ce=2.0833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.448987162206322e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024626823142170906, 'last_eos_top1': 0} +step=200000 lr=6.000000e-04 loss=2.0627 ce=2.0627 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.224479643104132e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001933229505084455, 'last_eos_top1': 0} +step=200050 lr=6.000000e-04 loss=2.5652 ce=2.5652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.46284671954345e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013950024731457233, 'last_eos_top1': 0} +step=200100 lr=6.000000e-04 loss=2.1187 ce=2.1187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.279261505260365e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001600281335413456, 'last_eos_top1': 0} +step=200150 lr=6.000000e-04 loss=2.6803 ce=2.6803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0943434972432442e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019298532279208302, 'last_eos_top1': 0} +step=200200 lr=6.000000e-04 loss=2.9508 ce=2.9508 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.271450835512951e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008567168843001127, 'last_eos_top1': 0} +step=200250 lr=6.000000e-04 loss=2.6867 ce=2.6867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4823907804384362e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010674273362383246, 'last_eos_top1': 0} +step=200300 lr=6.000000e-04 loss=2.8061 ce=2.8061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4920170542609412e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014190267538651824, 'last_eos_top1': 0} +step=200350 lr=6.000000e-04 loss=2.6226 ce=2.6226 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.43851591728162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019021965563297272, 'last_eos_top1': 0} +step=200400 lr=6.000000e-04 loss=2.2714 ce=2.2714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.054976544575766e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003226575907319784, 'last_eos_top1': 0} +step=200450 lr=6.000000e-04 loss=2.4324 ce=2.4324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.308360075810924e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022599021904170513, 'last_eos_top1': 0} +step=200500 lr=6.000000e-04 loss=2.1419 ce=2.1419 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.354502031375887e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026277536526322365, 'last_eos_top1': 0} +step=200550 lr=6.000000e-04 loss=2.6631 ce=2.6631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.420580914185848e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001697426661849022, 'last_eos_top1': 0} +step=200600 lr=6.000000e-04 loss=2.3185 ce=2.3185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.937323061923962e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014163819141685963, 'last_eos_top1': 0} +step=200650 lr=6.000000e-04 loss=1.7886 ce=1.7886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.453687198401894e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019681707490235567, 'last_eos_top1': 0} +step=200700 lr=6.000000e-04 loss=2.4718 ce=2.4718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.296116806770442e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013231136836111546, 'last_eos_top1': 0} +step=200750 lr=6.000000e-04 loss=2.5514 ce=2.5514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2121331565140281e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001059807138517499, 'last_eos_top1': 0} +step=200800 lr=6.000000e-04 loss=2.9308 ce=2.9308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.696440323139541e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006783946882933378, 'last_eos_top1': 0} +step=200850 lr=6.000000e-04 loss=2.8369 ce=2.8369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.29041881741432e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002766054240055382, 'last_eos_top1': 0} +step=200900 lr=6.000000e-04 loss=3.1987 ce=3.1987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.814840617153095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008376139448955655, 'last_eos_top1': 0} +step=200950 lr=6.000000e-04 loss=2.5568 ce=2.5568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0178966224193573e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010741607984527946, 'last_eos_top1': 0} +step=201000 lr=6.000000e-04 loss=2.1983 ce=2.1983 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.136299139645416e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005859662778675556, 'last_eos_top1': 0} +step=201050 lr=6.000000e-04 loss=2.4662 ce=2.4662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.792811058810912e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014118524268269539, 'last_eos_top1': 0} +step=201100 lr=6.000000e-04 loss=2.8372 ce=2.8372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.623810749151744e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010487402323633432, 'last_eos_top1': 0} +step=201150 lr=6.000000e-04 loss=2.6726 ce=2.6726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.314963421667926e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001574639230966568, 'last_eos_top1': 0} +step=201200 lr=6.000000e-04 loss=1.8833 ce=1.8833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.049338789016474e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007348940707743168, 'last_eos_top1': 0} +step=201250 lr=6.000000e-04 loss=1.9288 ce=1.9288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4107119568507187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006515938439406455, 'last_eos_top1': 0} +step=201300 lr=6.000000e-04 loss=2.6544 ce=2.6544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3669879990629852e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007790873060002923, 'last_eos_top1': 0} +step=201350 lr=6.000000e-04 loss=2.7887 ce=2.7887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4722092601004988e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008916318183764815, 'last_eos_top1': 0} +step=201400 lr=6.000000e-04 loss=2.2695 ce=2.2695 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.141429366834927e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005667394725605845, 'last_eos_top1': 0} +step=201450 lr=6.000000e-04 loss=2.5504 ce=2.5504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4323821233119816e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011351945577189326, 'last_eos_top1': 0} +step=201500 lr=6.000000e-04 loss=2.2973 ce=2.2973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.104614966607187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011649967636913061, 'last_eos_top1': 0} +step=201550 lr=6.000000e-04 loss=2.0322 ce=2.0322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.033283509372268e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017115373630076647, 'last_eos_top1': 0} +step=201600 lr=6.000000e-04 loss=2.3440 ce=2.3440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.399587957479525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010575170163065195, 'last_eos_top1': 0} +step=201650 lr=6.000000e-04 loss=2.4587 ce=2.4587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.873107966181124e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010067746043205261, 'last_eos_top1': 0} +step=201700 lr=6.000000e-04 loss=2.6512 ce=2.6512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9763541786232963e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001381652895361185, 'last_eos_top1': 0} +step=201750 lr=6.000000e-04 loss=2.2494 ce=2.2494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0834691238414962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001956881955265999, 'last_eos_top1': 0} +step=201800 lr=6.000000e-04 loss=2.9533 ce=2.9533 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.047505510447081e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027101202867925167, 'last_eos_top1': 0} +step=201850 lr=6.000000e-04 loss=1.9843 ce=1.9843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.91491469903849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001991241704672575, 'last_eos_top1': 0} +step=201900 lr=6.000000e-04 loss=2.7163 ce=2.7163 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2254669854883105e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013759119901806116, 'last_eos_top1': 0} +step=201950 lr=6.000000e-04 loss=2.8441 ce=2.8441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.277887223404832e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007210444891825318, 'last_eos_top1': 0} +step=202000 lr=6.000000e-04 loss=2.1230 ce=2.1230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0995920092682354e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020997263491153717, 'last_eos_top1': 0} +step=202050 lr=6.000000e-04 loss=2.3822 ce=2.3822 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.14222255232744e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003009129548445344, 'last_eos_top1': 0} +step=202100 lr=6.000000e-04 loss=2.7785 ce=2.7785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1061102668463718e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024785897694528103, 'last_eos_top1': 0} +step=202150 lr=6.000000e-04 loss=3.0492 ce=3.0492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.490937034366652e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019327017944306135, 'last_eos_top1': 0} +step=202200 lr=6.000000e-04 loss=2.2872 ce=2.2872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.306921128882095e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00276769557967782, 'last_eos_top1': 0} +step=202250 lr=6.000000e-04 loss=2.7077 ce=2.7077 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.673160558217205e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001987393479794264, 'last_eos_top1': 0} +step=202300 lr=6.000000e-04 loss=2.1953 ce=2.1953 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.81597703078296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002934593241661787, 'last_eos_top1': 0} +step=202350 lr=6.000000e-04 loss=2.6001 ce=2.6001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3433702810725663e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002920433646067977, 'last_eos_top1': 0} +step=202400 lr=6.000000e-04 loss=2.8309 ce=2.8309 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.920160219829995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026080962270498276, 'last_eos_top1': 0} +step=202450 lr=6.000000e-04 loss=2.5178 ce=2.5178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1257083315285854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031000690069049597, 'last_eos_top1': 0} +step=202500 lr=6.000000e-04 loss=2.8003 ce=2.8003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.49192110763397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003926405683159828, 'last_eos_top1': 0} +step=202550 lr=6.000000e-04 loss=2.5193 ce=2.5193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8182619896833785e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00306529994122684, 'last_eos_top1': 0} +step=202600 lr=6.000000e-04 loss=2.4296 ce=2.4296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2420160601323005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032332278788089752, 'last_eos_top1': 0} +step=202650 lr=6.000000e-04 loss=2.5837 ce=2.5837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.500769253150793e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023354075383394957, 'last_eos_top1': 0} +step=202700 lr=6.000000e-04 loss=2.1464 ce=2.1464 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.137799351155991e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023808362893760204, 'last_eos_top1': 0} +step=202750 lr=6.000000e-04 loss=2.0403 ce=2.0403 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.004324520996306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028336208779364824, 'last_eos_top1': 0} +step=202800 lr=6.000000e-04 loss=2.0055 ce=2.0055 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.23212259192951e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002928046975284815, 'last_eos_top1': 0} +step=202850 lr=6.000000e-04 loss=2.6221 ce=2.6221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1175472536706366e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020849225111305714, 'last_eos_top1': 0} +step=202900 lr=6.000000e-04 loss=2.8169 ce=2.8169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.724765520018991e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023736394941806793, 'last_eos_top1': 0} +step=202950 lr=6.000000e-04 loss=2.4001 ce=2.4001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.135518899303861e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016809756634756923, 'last_eos_top1': 0} +step=203000 lr=6.000000e-04 loss=2.6597 ce=2.6597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8443954104441218e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020949137397110462, 'last_eos_top1': 0} +step=203050 lr=6.000000e-04 loss=2.3500 ce=2.3500 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0209223546553403e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016009148675948381, 'last_eos_top1': 0} +step=203100 lr=6.000000e-04 loss=2.8031 ce=2.8031 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9007654777378775e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009705288102850318, 'last_eos_top1': 0} +step=203150 lr=6.000000e-04 loss=2.8750 ce=2.8750 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2346298717602622e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002404254861176014, 'last_eos_top1': 0} +step=203200 lr=6.000000e-04 loss=2.6995 ce=2.6995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.387532605207525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014107883907854557, 'last_eos_top1': 0} +step=203250 lr=6.000000e-04 loss=2.4042 ce=2.4042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0822782314789947e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022173149045556784, 'last_eos_top1': 0} +step=203300 lr=6.000000e-04 loss=2.2459 ce=2.2459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1428254765633028e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011674129636958241, 'last_eos_top1': 0} +step=203350 lr=6.000000e-04 loss=2.6361 ce=2.6361 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.1197439461247995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018983444897457957, 'last_eos_top1': 0} +step=203400 lr=6.000000e-04 loss=2.9807 ce=2.9807 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0123396350536495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002677976619452238, 'last_eos_top1': 0} +step=203450 lr=6.000000e-04 loss=2.4308 ce=2.4308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3954289897810668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003190233837813139, 'last_eos_top1': 0} +step=203500 lr=6.000000e-04 loss=2.6038 ce=2.6038 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7324593500234187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020227800123393536, 'last_eos_top1': 0} +step=203550 lr=6.000000e-04 loss=2.1848 ce=2.1848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.9662279454641975e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014484110288321972, 'last_eos_top1': 0} +step=203600 lr=6.000000e-04 loss=2.2163 ce=2.2163 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4295909750217106e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013726471224799752, 'last_eos_top1': 0} +step=203650 lr=6.000000e-04 loss=2.3136 ce=2.3136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8214800446876325e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019205277785658836, 'last_eos_top1': 0} +step=203700 lr=6.000000e-04 loss=2.8199 ce=2.8199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.457543688360602e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002220936119556427, 'last_eos_top1': 0} +step=203750 lr=6.000000e-04 loss=3.0369 ce=3.0369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.292190624255454e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031130351126194, 'last_eos_top1': 0} +step=203800 lr=6.000000e-04 loss=2.9976 ce=2.9976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.2818900156999e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035510214511305094, 'last_eos_top1': 0} +step=203850 lr=6.000000e-04 loss=2.7242 ce=2.7242 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.921021511312574e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003845999948680401, 'last_eos_top1': 0} +step=203900 lr=6.000000e-04 loss=2.5936 ce=2.5936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.325073056563269e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004149735439568758, 'last_eos_top1': 0} +step=203950 lr=6.000000e-04 loss=2.2017 ce=2.2017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7617447156226262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004043801687657833, 'last_eos_top1': 0} +step=204000 lr=6.000000e-04 loss=2.4086 ce=2.4086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5135459761950187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003711957950145006, 'last_eos_top1': 0} +step=204050 lr=6.000000e-04 loss=1.9095 ce=1.9095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3599936210084707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036570937372744083, 'last_eos_top1': 0} +step=204100 lr=6.000000e-04 loss=2.1506 ce=2.1506 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.972404091560747e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00224504666402936, 'last_eos_top1': 0} +step=204150 lr=6.000000e-04 loss=2.7099 ce=2.7099 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.465398761676624e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035283113829791546, 'last_eos_top1': 0} +step=204200 lr=6.000000e-04 loss=2.6302 ce=2.6302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.935461664805189e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030520048458129168, 'last_eos_top1': 0} +step=204250 lr=6.000000e-04 loss=2.5320 ce=2.5320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.631151098990813e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002869900781661272, 'last_eos_top1': 0} +step=204300 lr=6.000000e-04 loss=2.3166 ce=2.3166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.815719072823413e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027714362367987633, 'last_eos_top1': 0} +step=204350 lr=6.000000e-04 loss=2.5002 ce=2.5002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.321255336340982e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019150732550770044, 'last_eos_top1': 0} +step=204400 lr=6.000000e-04 loss=2.3112 ce=2.3112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.280338417738676e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019740904681384563, 'last_eos_top1': 0} +step=204450 lr=6.000000e-04 loss=2.6327 ce=2.6327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.828617865219712e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019222705159336329, 'last_eos_top1': 0} +step=204500 lr=6.000000e-04 loss=2.4860 ce=2.4860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0765793376776855e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021674928721040487, 'last_eos_top1': 0} +step=204550 lr=6.000000e-04 loss=3.3523 ce=3.3523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.217452129116282e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021433301735669374, 'last_eos_top1': 0} +step=204600 lr=6.000000e-04 loss=2.4993 ce=2.4993 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.564833812357392e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026057232171297073, 'last_eos_top1': 0} +step=204650 lr=6.000000e-04 loss=2.7601 ce=2.7601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4332665159599856e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0041713593527674675, 'last_eos_top1': 0} +step=204700 lr=6.000000e-04 loss=2.4402 ce=2.4402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0746487532742321e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004103686660528183, 'last_eos_top1': 0} +step=204750 lr=6.000000e-04 loss=2.6335 ce=2.6335 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8260629076394252e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003010317450389266, 'last_eos_top1': 0} +step=204800 lr=6.000000e-04 loss=2.2080 ce=2.2080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.567834210116416e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035883861128240824, 'last_eos_top1': 0} +step=204850 lr=6.000000e-04 loss=2.7815 ce=2.7815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.501696872059256e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036649005487561226, 'last_eos_top1': 0} +step=204900 lr=6.000000e-04 loss=2.5987 ce=2.5987 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5652969523216598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00359516404569149, 'last_eos_top1': 0} +step=204950 lr=6.000000e-04 loss=2.7218 ce=2.7218 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.504021383880172e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002272174693644047, 'last_eos_top1': 0} +step=205000 lr=6.000000e-04 loss=2.6252 ce=2.6252 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1342070138198324e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022281864657998085, 'last_eos_top1': 0} +step=205050 lr=6.000000e-04 loss=2.6501 ce=2.6501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.619454426167067e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029236478731036186, 'last_eos_top1': 0} +step=205100 lr=6.000000e-04 loss=2.8533 ce=2.8533 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.358545614493778e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017949772300198674, 'last_eos_top1': 0} +step=205150 lr=6.000000e-04 loss=2.4287 ce=2.4287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.167282663227525e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004533257451839745, 'last_eos_top1': 0} +step=205200 lr=6.000000e-04 loss=3.4167 ce=3.4167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.33618196338648e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006416093674488366, 'last_eos_top1': 0} +step=205250 lr=6.000000e-04 loss=2.1732 ce=2.1732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.033919736277312e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012393895303830504, 'last_eos_top1': 0} +step=205300 lr=6.000000e-04 loss=2.6752 ce=2.6752 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9371365851839073e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014709343668073416, 'last_eos_top1': 0} +step=205350 lr=6.000000e-04 loss=1.7538 ce=1.7538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0572141036391258e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001372199971228838, 'last_eos_top1': 0} +step=205400 lr=6.000000e-04 loss=2.7227 ce=2.7227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0364323316025548e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009338065865449607, 'last_eos_top1': 0} +step=205450 lr=6.000000e-04 loss=2.6155 ce=2.6155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.468283572758082e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009133997955359519, 'last_eos_top1': 0} +step=205500 lr=6.000000e-04 loss=3.2430 ce=3.2430 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.054836143855937e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007320106960833073, 'last_eos_top1': 0} +step=205550 lr=6.000000e-04 loss=2.0410 ce=2.0410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2674941899604164e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000789306010119617, 'last_eos_top1': 0} +step=205600 lr=6.000000e-04 loss=2.6121 ce=2.6121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.702355444256682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009142229100689292, 'last_eos_top1': 0} +step=205650 lr=6.000000e-04 loss=2.4815 ce=2.4815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.470733852533158e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009899336146190763, 'last_eos_top1': 0} +step=205700 lr=6.000000e-04 loss=2.1889 ce=2.1889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.776821954990737e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008866718853823841, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3300000 rows=3291809 buffered=8191 part=openwebtext-t5 +step=205750 lr=6.000000e-04 loss=2.4686 ce=2.4686 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4195118637871929e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00090157485101372, 'last_eos_top1': 0} +step=205800 lr=6.000000e-04 loss=2.7157 ce=2.7157 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5476038242923096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004466544196475297, 'last_eos_top1': 0} +step=205850 lr=6.000000e-04 loss=2.3456 ce=2.3456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0291721991961822e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006223131204023957, 'last_eos_top1': 0} +step=205900 lr=6.000000e-04 loss=2.2832 ce=2.2832 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7139114788733423e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005887462175451219, 'last_eos_top1': 0} +step=205950 lr=6.000000e-04 loss=2.2828 ce=2.2828 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.371815142192645e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006492640823125839, 'last_eos_top1': 0} +step=206000 lr=6.000000e-04 loss=2.5894 ce=2.5894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9582825188990682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007975457701832056, 'last_eos_top1': 0} +step=206050 lr=6.000000e-04 loss=2.6591 ce=2.6591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2585342119564302e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005797614576295018, 'last_eos_top1': 0} +step=206100 lr=6.000000e-04 loss=2.8074 ce=2.8074 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3343382306629792e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012166328961029649, 'last_eos_top1': 0} +step=206150 lr=6.000000e-04 loss=2.2872 ce=2.2872 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.605123755638488e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006247177952900529, 'last_eos_top1': 0} +step=206200 lr=6.000000e-04 loss=2.0851 ce=2.0851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.746880666876677e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006335936486721039, 'last_eos_top1': 0} +step=206250 lr=6.000000e-04 loss=3.1592 ce=3.1592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3476079402607866e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007051419233903289, 'last_eos_top1': 0} +step=206300 lr=6.000000e-04 loss=2.5855 ce=2.5855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0323752576368861e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004871122073382139, 'last_eos_top1': 0} +step=206350 lr=6.000000e-04 loss=2.5280 ce=2.5280 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8587395970826037e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006222233641892672, 'last_eos_top1': 0} +step=206400 lr=6.000000e-04 loss=2.7533 ce=2.7533 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3961125407367945e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007595497881993651, 'last_eos_top1': 0} +step=206450 lr=6.000000e-04 loss=2.8563 ce=2.8563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.254906576126814e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010054688900709152, 'last_eos_top1': 0} +step=206500 lr=6.000000e-04 loss=1.6289 ce=1.6289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9856214395258576e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010299712885171175, 'last_eos_top1': 0} +step=206550 lr=6.000000e-04 loss=2.9973 ce=2.9973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.582036697480362e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013423571363091469, 'last_eos_top1': 0} +step=206600 lr=6.000000e-04 loss=2.7875 ce=2.7875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.343247954035178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001062752678990364, 'last_eos_top1': 0} +step=206650 lr=6.000000e-04 loss=2.6146 ce=2.6146 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.74516171682626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010589929297566414, 'last_eos_top1': 0} +step=206700 lr=6.000000e-04 loss=2.5721 ce=2.5721 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.811873158265371e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011115818051621318, 'last_eos_top1': 0} +step=206750 lr=6.000000e-04 loss=2.3523 ce=2.3523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.21725006971974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020523760467767715, 'last_eos_top1': 0} +step=206800 lr=6.000000e-04 loss=2.9999 ce=2.9999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.255934668937698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022964822128415108, 'last_eos_top1': 0} +step=206850 lr=6.000000e-04 loss=2.4490 ce=2.4490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0424066203995608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001986200688406825, 'last_eos_top1': 0} +step=206900 lr=6.000000e-04 loss=2.4138 ce=2.4138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0668893386027776e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001359814079478383, 'last_eos_top1': 0} +step=206950 lr=6.000000e-04 loss=2.3140 ce=2.3140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.972820948343724e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018920016009360552, 'last_eos_top1': 0} +step=207000 lr=6.000000e-04 loss=2.3321 ce=2.3321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.061319537460804e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016304589807987213, 'last_eos_top1': 0} +step=207050 lr=6.000000e-04 loss=2.2815 ce=2.2815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9716562746907584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001601383788511157, 'last_eos_top1': 0} +step=207100 lr=6.000000e-04 loss=2.4325 ce=2.4325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.172290447168052e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015079056611284614, 'last_eos_top1': 0} +step=207150 lr=6.000000e-04 loss=2.3800 ce=2.3800 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.58360305451788e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020653510000556707, 'last_eos_top1': 0} +step=207200 lr=6.000000e-04 loss=2.9592 ce=2.9592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.285618757014163e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002924699103459716, 'last_eos_top1': 0} +step=207250 lr=6.000000e-04 loss=2.4450 ce=2.4450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8463917513145134e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002353168558329344, 'last_eos_top1': 0} +step=207300 lr=6.000000e-04 loss=2.4141 ce=2.4141 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.376238105876837e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002065110020339489, 'last_eos_top1': 0} +step=207350 lr=6.000000e-04 loss=2.0640 ce=2.0640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6951718609780073e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022964803501963615, 'last_eos_top1': 0} +step=207400 lr=6.000000e-04 loss=2.1748 ce=2.1748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0281796676281374e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016016812296584249, 'last_eos_top1': 0} +step=207450 lr=6.000000e-04 loss=2.6905 ce=2.6905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.203124520543497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011810349533334374, 'last_eos_top1': 0} +step=207500 lr=6.000000e-04 loss=2.0446 ce=2.0446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.064892371185124e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002007451606914401, 'last_eos_top1': 0} +step=207550 lr=6.000000e-04 loss=2.6659 ce=2.6659 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.099785191006958e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001498954021371901, 'last_eos_top1': 0} +step=207600 lr=6.000000e-04 loss=2.3997 ce=2.3997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4485804058494978e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030557080172002316, 'last_eos_top1': 0} +step=207650 lr=6.000000e-04 loss=2.1259 ce=2.1259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2000346032436937e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031157643534243107, 'last_eos_top1': 0} +step=207700 lr=6.000000e-04 loss=2.8707 ce=2.8707 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.538691867608577e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002691442845389247, 'last_eos_top1': 0} +step=207750 lr=6.000000e-04 loss=3.3574 ce=3.3574 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4152737094264012e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030962256714701653, 'last_eos_top1': 0} +step=207800 lr=6.000000e-04 loss=2.3262 ce=2.3262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.661241736263037e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017297356389462948, 'last_eos_top1': 0} +step=207850 lr=6.000000e-04 loss=2.6492 ce=2.6492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.384235803125193e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019307925831526518, 'last_eos_top1': 0} +step=207900 lr=6.000000e-04 loss=2.2521 ce=2.2521 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.327260881429538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002299259416759014, 'last_eos_top1': 0} +step=207950 lr=6.000000e-04 loss=2.0470 ce=2.0470 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.454268936475273e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003083731746301055, 'last_eos_top1': 0} +step=208000 lr=6.000000e-04 loss=2.6526 ce=2.6526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2216249160701409e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033237654715776443, 'last_eos_top1': 0} +step=208050 lr=6.000000e-04 loss=2.4067 ce=2.4067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.659274170786375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022838732693344355, 'last_eos_top1': 0} +step=208100 lr=6.000000e-04 loss=2.8770 ce=2.8770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.751378583023325e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023857890628278255, 'last_eos_top1': 0} +step=208150 lr=6.000000e-04 loss=2.4343 ce=2.4343 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2268701539142057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033432492054998875, 'last_eos_top1': 0} +step=208200 lr=6.000000e-04 loss=2.5110 ce=2.5110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8001854186877608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002455594018101692, 'last_eos_top1': 0} +step=208250 lr=6.000000e-04 loss=3.1579 ce=3.1579 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.654452929273248e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029389660339802504, 'last_eos_top1': 0} +step=208300 lr=6.000000e-04 loss=2.4406 ce=2.4406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0500221833353862e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034376278053969145, 'last_eos_top1': 0} +step=208350 lr=6.000000e-04 loss=2.5143 ce=2.5143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.82553137873765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003011156804859638, 'last_eos_top1': 0} +step=208400 lr=6.000000e-04 loss=2.6985 ce=2.6985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1250855095568113e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002670350018888712, 'last_eos_top1': 0} +step=208450 lr=6.000000e-04 loss=2.9929 ce=2.9929 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5234623788273893e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002537289634346962, 'last_eos_top1': 0} +step=208500 lr=6.000000e-04 loss=2.7010 ce=2.7010 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2974271157872863e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00306318374350667, 'last_eos_top1': 0} +step=208550 lr=6.000000e-04 loss=2.1621 ce=2.1621 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.450653017964214e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018153057899326086, 'last_eos_top1': 0} +step=208600 lr=6.000000e-04 loss=2.5481 ce=2.5481 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3500954082701355e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002272506710141897, 'last_eos_top1': 0} +step=208650 lr=6.000000e-04 loss=2.1683 ce=2.1683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.543942724121734e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022720058914273977, 'last_eos_top1': 0} +step=208700 lr=6.000000e-04 loss=2.5976 ce=2.5976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5941648598527536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002347018802538514, 'last_eos_top1': 0} +step=208750 lr=6.000000e-04 loss=2.7417 ce=2.7417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0518977988976985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001699613407254219, 'last_eos_top1': 0} +step=208800 lr=6.000000e-04 loss=2.6482 ce=2.6482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.599750228517223e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009097001748159528, 'last_eos_top1': 0} +step=208850 lr=6.000000e-04 loss=2.2884 ce=2.2884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0957344784401357e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001907814061269164, 'last_eos_top1': 0} +step=208900 lr=6.000000e-04 loss=2.7905 ce=2.7905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.090179911348969e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011769182747229934, 'last_eos_top1': 0} +step=208950 lr=6.000000e-04 loss=2.3827 ce=2.3827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.66443406266626e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017865380505099893, 'last_eos_top1': 0} +step=209000 lr=6.000000e-04 loss=1.9076 ce=1.9076 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.485332728014328e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019228151068091393, 'last_eos_top1': 0} +step=209050 lr=6.000000e-04 loss=2.3856 ce=2.3856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.728205825202167e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002551984041929245, 'last_eos_top1': 0} +step=209100 lr=6.000000e-04 loss=2.7598 ce=2.7598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.123622314480599e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001894240966066718, 'last_eos_top1': 0} +step=209150 lr=6.000000e-04 loss=2.4930 ce=2.4930 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0260439921694342e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001515320735052228, 'last_eos_top1': 0} +step=209200 lr=6.000000e-04 loss=2.7035 ce=2.7035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.521270840195939e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013747350312769413, 'last_eos_top1': 0} +step=209250 lr=6.000000e-04 loss=2.8861 ce=2.8861 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4017543738591485e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012408164329826832, 'last_eos_top1': 0} +step=209300 lr=6.000000e-04 loss=2.7017 ce=2.7017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2240601790836081e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020287614315748215, 'last_eos_top1': 0} +step=209350 lr=6.000000e-04 loss=2.6274 ce=2.6274 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.102172154787695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012843681033700705, 'last_eos_top1': 0} +step=209400 lr=6.000000e-04 loss=2.9411 ce=2.9411 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.30357466332498e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008914296631701291, 'last_eos_top1': 0} +step=209450 lr=6.000000e-04 loss=2.5188 ce=2.5188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9821694877464324e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013922357466071844, 'last_eos_top1': 0} +step=209500 lr=6.000000e-04 loss=2.4824 ce=2.4824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.689555887831375e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015501382295042276, 'last_eos_top1': 0} +step=209550 lr=6.000000e-04 loss=2.3385 ce=2.3385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9087488908553496e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001397275715135038, 'last_eos_top1': 0} +step=209600 lr=6.000000e-04 loss=2.6547 ce=2.6547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4666804721346125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012733211042359471, 'last_eos_top1': 0} +step=209650 lr=6.000000e-04 loss=2.3296 ce=2.3296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.150671419571154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001442690845578909, 'last_eos_top1': 0} +step=209700 lr=6.000000e-04 loss=2.5051 ce=2.5051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0238429240416735e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009624387603253126, 'last_eos_top1': 0} +step=209750 lr=6.000000e-04 loss=2.6480 ce=2.6480 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1287571144057438e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013547517592087388, 'last_eos_top1': 0} +step=209800 lr=6.000000e-04 loss=2.6477 ce=2.6477 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7512771339388564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011845018016174436, 'last_eos_top1': 0} +step=209850 lr=6.000000e-04 loss=2.0754 ce=2.0754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.904104106477462e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009555003489367664, 'last_eos_top1': 0} +step=209900 lr=6.000000e-04 loss=2.1653 ce=2.1653 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4459862945368513e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008942385320551693, 'last_eos_top1': 0} +step=209950 lr=6.000000e-04 loss=2.2737 ce=2.2737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.5648187046463136e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005488331662490964, 'last_eos_top1': 0} +step=210000 lr=6.000000e-04 loss=2.3004 ce=2.3004 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.30528460937785e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009305569110438228, 'last_eos_top1': 0} +step=210050 lr=6.000000e-04 loss=1.8425 ce=1.8425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0149404008407146e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001902377000078559, 'last_eos_top1': 0} +step=210100 lr=6.000000e-04 loss=2.8878 ce=2.8878 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5765996067784727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017369400011375546, 'last_eos_top1': 0} +step=210150 lr=6.000000e-04 loss=2.6232 ce=2.6232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1971693058731034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025068398099392653, 'last_eos_top1': 0} +step=210200 lr=6.000000e-04 loss=2.1927 ce=2.1927 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4214283510227688e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002220946131274104, 'last_eos_top1': 0} +step=210250 lr=6.000000e-04 loss=2.8552 ce=2.8552 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1407360943849199e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022022002376616, 'last_eos_top1': 0} +step=210300 lr=6.000000e-04 loss=2.5516 ce=2.5516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.76470657606842e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015891310758888721, 'last_eos_top1': 0} +step=210350 lr=6.000000e-04 loss=2.5717 ce=2.5717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0325771654606797e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008624843321740627, 'last_eos_top1': 0} +step=210400 lr=6.000000e-04 loss=2.6367 ce=2.6367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.473142492613988e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007273615337908268, 'last_eos_top1': 0} +step=210450 lr=6.000000e-04 loss=2.6068 ce=2.6068 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.416527063876856e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017402900848537683, 'last_eos_top1': 0} +step=210500 lr=6.000000e-04 loss=2.4024 ce=2.4024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.73648424303974e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001482432708144188, 'last_eos_top1': 0} +step=210550 lr=6.000000e-04 loss=2.5337 ce=2.5337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3696175048826262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002231436548754573, 'last_eos_top1': 0} +step=210600 lr=6.000000e-04 loss=2.3185 ce=2.3185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2718264517607167e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023869737051427364, 'last_eos_top1': 0} +step=210650 lr=6.000000e-04 loss=2.6363 ce=2.6363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5443594747921452e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027215289883315563, 'last_eos_top1': 0} +step=210700 lr=6.000000e-04 loss=2.4451 ce=2.4451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.88583890121663e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003331215586513281, 'last_eos_top1': 0} +step=210750 lr=6.000000e-04 loss=2.8378 ce=2.8378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5957195248338394e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004238964058458805, 'last_eos_top1': 0} +step=210800 lr=6.000000e-04 loss=2.8337 ce=2.8337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6114638128783554e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004361780360341072, 'last_eos_top1': 0} +step=210850 lr=6.000000e-04 loss=2.1952 ce=2.1952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7723314158502035e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034775491803884506, 'last_eos_top1': 0} +step=210900 lr=6.000000e-04 loss=2.2903 ce=2.2903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6443551430711523e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028965375386178493, 'last_eos_top1': 0} +step=210950 lr=6.000000e-04 loss=2.7176 ce=2.7176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3763770766672678e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023904643021523952, 'last_eos_top1': 0} +step=211000 lr=6.000000e-04 loss=1.9112 ce=1.9112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.181349893566221e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029306430369615555, 'last_eos_top1': 0} +step=211050 lr=6.000000e-04 loss=2.7087 ce=2.7087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.286576662096195e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002219672780483961, 'last_eos_top1': 0} +step=211100 lr=6.000000e-04 loss=2.4232 ce=2.4232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6395966667914763e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003696745727211237, 'last_eos_top1': 0} +step=211150 lr=6.000000e-04 loss=2.8977 ce=2.8977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9216166593832895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004026381764560938, 'last_eos_top1': 0} +step=211200 lr=6.000000e-04 loss=2.1004 ce=2.1004 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.090259143500589e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00300733745098114, 'last_eos_top1': 0} +step=211250 lr=6.000000e-04 loss=3.1027 ce=3.1027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.050236258190125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002944672480225563, 'last_eos_top1': 0} +step=211300 lr=6.000000e-04 loss=2.6186 ce=2.6186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.823101203830447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002847283612936735, 'last_eos_top1': 0} +step=211350 lr=6.000000e-04 loss=2.1800 ce=2.1800 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.063081253960263e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036279764026403427, 'last_eos_top1': 0} +step=211400 lr=6.000000e-04 loss=1.8196 ce=1.8196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2420408893376589e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036148554645478725, 'last_eos_top1': 0} +step=211450 lr=6.000000e-04 loss=2.7590 ce=2.7590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7269940144615248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002885957947000861, 'last_eos_top1': 0} +step=211500 lr=6.000000e-04 loss=2.6999 ce=2.6999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.527849680220243e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020115531515330076, 'last_eos_top1': 0} +step=211550 lr=6.000000e-04 loss=2.6209 ce=2.6209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.541792911069933e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002694241236895323, 'last_eos_top1': 0} +step=211600 lr=6.000000e-04 loss=2.6846 ce=2.6846 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3369340194913093e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025839507579803467, 'last_eos_top1': 0} +step=211650 lr=6.000000e-04 loss=2.2869 ce=2.2869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6283393051708117e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020283223129808903, 'last_eos_top1': 0} +step=211700 lr=6.000000e-04 loss=2.7324 ce=2.7324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3817043509334326e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014273839769884944, 'last_eos_top1': 0} +step=211750 lr=6.000000e-04 loss=2.5637 ce=2.5637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.134049388288986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009405348100699484, 'last_eos_top1': 0} +step=211800 lr=6.000000e-04 loss=2.2170 ce=2.2170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.875622835243121e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015755584463477135, 'last_eos_top1': 0} +step=211850 lr=6.000000e-04 loss=2.9351 ce=2.9351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.155604857369326e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015546797076240182, 'last_eos_top1': 0} +step=211900 lr=6.000000e-04 loss=2.3878 ce=2.3878 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.41053258429747e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016166290733963251, 'last_eos_top1': 0} +step=211950 lr=6.000000e-04 loss=2.1120 ce=2.1120 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.55261805554619e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00121394544839859, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=2 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3400000 rows=3391809 buffered=8191 part=openwebtext-t5 +step=212000 lr=6.000000e-04 loss=2.7997 ce=2.7997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.814437827211805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013961008517071605, 'last_eos_top1': 0} +step=212050 lr=6.000000e-04 loss=2.5606 ce=2.5606 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.754018952255137e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002400070196017623, 'last_eos_top1': 0} +step=212100 lr=6.000000e-04 loss=1.8751 ce=1.8751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.618791980552487e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022698116954416037, 'last_eos_top1': 0} +step=212150 lr=6.000000e-04 loss=1.9902 ce=1.9902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7621614359959494e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026347022503614426, 'last_eos_top1': 0} +step=212200 lr=6.000000e-04 loss=2.7694 ce=2.7694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1636133422143757e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031967866234481335, 'last_eos_top1': 0} +step=212250 lr=6.000000e-04 loss=2.3842 ce=2.3842 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.930668966262601e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002407887950539589, 'last_eos_top1': 0} +step=212300 lr=6.000000e-04 loss=2.2785 ce=2.2785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3861682570423e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017112120985984802, 'last_eos_top1': 0} +step=212350 lr=6.000000e-04 loss=2.7460 ce=2.7460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.518762115272693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016845657955855131, 'last_eos_top1': 0} +step=212400 lr=6.000000e-04 loss=2.3566 ce=2.3566 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.922773198108189e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016769205685704947, 'last_eos_top1': 0} +step=212450 lr=6.000000e-04 loss=2.3894 ce=2.3894 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2636273822863586e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014163779560476542, 'last_eos_top1': 0} +step=212500 lr=6.000000e-04 loss=2.9174 ce=2.9174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1069534593843855e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018106151837855577, 'last_eos_top1': 0} +step=212550 lr=6.000000e-04 loss=2.7947 ce=2.7947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6892608982743695e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025018085725605488, 'last_eos_top1': 0} +step=212600 lr=6.000000e-04 loss=2.2451 ce=2.2451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.905794644902926e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008892660262063146, 'last_eos_top1': 0} +step=212650 lr=6.000000e-04 loss=2.6470 ce=2.6470 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.579974534455687e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005740498309023678, 'last_eos_top1': 0} +step=212700 lr=6.000000e-04 loss=2.5436 ce=2.5436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.521619757724693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010571496095508337, 'last_eos_top1': 0} +step=212750 lr=6.000000e-04 loss=2.5104 ce=2.5104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.744094884605147e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018617722671478987, 'last_eos_top1': 0} +step=212800 lr=6.000000e-04 loss=2.4834 ce=2.4834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2668991985265166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023732678964734077, 'last_eos_top1': 0} +step=212850 lr=6.000000e-04 loss=3.2558 ce=3.2558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6246605810010806e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015111162792891264, 'last_eos_top1': 0} +step=212900 lr=6.000000e-04 loss=2.8691 ce=2.8691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1492937119328417e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023956827353686094, 'last_eos_top1': 0} +step=212950 lr=6.000000e-04 loss=2.2473 ce=2.2473 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4244069461710751e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018376025836914778, 'last_eos_top1': 0} +step=213000 lr=6.000000e-04 loss=2.6769 ce=2.6769 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1860105587402359e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019886470399796963, 'last_eos_top1': 0} +step=213050 lr=6.000000e-04 loss=2.4088 ce=2.4088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1832616766914725e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018154812278226018, 'last_eos_top1': 0} +step=213100 lr=6.000000e-04 loss=2.2956 ce=2.2956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.771861752786208e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001658485271036625, 'last_eos_top1': 0} +step=213150 lr=6.000000e-04 loss=2.4739 ce=2.4739 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4533739886246622e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017867557471618056, 'last_eos_top1': 0} +step=213200 lr=6.000000e-04 loss=2.6015 ce=2.6015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.595504939672537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012326538562774658, 'last_eos_top1': 0} +step=213250 lr=6.000000e-04 loss=2.3758 ce=2.3758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.577093467465602e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000920096761547029, 'last_eos_top1': 0} +step=213300 lr=6.000000e-04 loss=2.7033 ce=2.7033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8700282453210093e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008358339546248317, 'last_eos_top1': 0} +step=213350 lr=6.000000e-04 loss=3.1249 ce=3.1249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9770277503994294e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015748648438602686, 'last_eos_top1': 0} +step=213400 lr=6.000000e-04 loss=1.8909 ce=1.8909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2454801435524132e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011810727883130312, 'last_eos_top1': 0} +step=213450 lr=6.000000e-04 loss=2.9525 ce=2.9525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.832112143863924e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012920283479616046, 'last_eos_top1': 0} +step=213500 lr=6.000000e-04 loss=2.3454 ce=2.3454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1223927685932722e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001443374902009964, 'last_eos_top1': 0} +step=213550 lr=6.000000e-04 loss=2.2217 ce=2.2217 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9882496417267248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001970520708709955, 'last_eos_top1': 0} +step=213600 lr=6.000000e-04 loss=2.3051 ce=2.3051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.925078873289749e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001131344004534185, 'last_eos_top1': 0} +step=213650 lr=6.000000e-04 loss=2.3546 ce=2.3546 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.846119736612309e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010113788302987814, 'last_eos_top1': 0} +step=213700 lr=6.000000e-04 loss=2.4509 ce=2.4509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.652481821831316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009091091342270374, 'last_eos_top1': 0} +step=213750 lr=6.000000e-04 loss=2.4034 ce=2.4034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0080691026814748e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008918477105908096, 'last_eos_top1': 0} +step=213800 lr=6.000000e-04 loss=2.6977 ce=2.6977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.074224413372576e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006279268418438733, 'last_eos_top1': 0} +step=213850 lr=6.000000e-04 loss=2.0890 ce=2.0890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0284454219799954e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008720209589228034, 'last_eos_top1': 0} +step=213900 lr=6.000000e-04 loss=2.0988 ce=2.0988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0685157576517668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008686439832672477, 'last_eos_top1': 0} +step=213950 lr=6.000000e-04 loss=2.6896 ce=2.6896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1361345968907699e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012910037767142057, 'last_eos_top1': 0} +step=214000 lr=6.000000e-04 loss=2.2347 ce=2.2347 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0929554264293984e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005290787667036057, 'last_eos_top1': 0} +step=214050 lr=6.000000e-04 loss=2.4501 ce=2.4501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.151689482678194e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006812120554968715, 'last_eos_top1': 0} +step=214100 lr=6.000000e-04 loss=2.7397 ce=2.7397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.391942745016422e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011596755357459188, 'last_eos_top1': 0} +step=214150 lr=6.000000e-04 loss=2.5407 ce=2.5407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.889392691140529e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008372496231459081, 'last_eos_top1': 0} +step=214200 lr=6.000000e-04 loss=2.4951 ce=2.4951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.022807378438301e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007534181931987405, 'last_eos_top1': 0} +step=214250 lr=6.000000e-04 loss=2.9677 ce=2.9677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.32512728770962e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00019329742644913495, 'last_eos_top1': 0} +step=214300 lr=6.000000e-04 loss=2.8436 ce=2.8436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8961882233270444e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044119509402662516, 'last_eos_top1': 0} +step=214350 lr=6.000000e-04 loss=2.5776 ce=2.5776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.754468919709325e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005646332865580916, 'last_eos_top1': 0} +step=214400 lr=6.000000e-04 loss=2.6391 ce=2.6391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.813184543512762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039325718535110354, 'last_eos_top1': 0} +step=214450 lr=6.000000e-04 loss=2.4573 ce=2.4573 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.21444451098796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000737731927074492, 'last_eos_top1': 0} +step=214500 lr=6.000000e-04 loss=1.9297 ce=1.9297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.348464866168797e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008161723962984979, 'last_eos_top1': 0} +step=214550 lr=6.000000e-04 loss=2.0944 ce=2.0944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4886949429637752e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008249012171290815, 'last_eos_top1': 0} +step=214600 lr=6.000000e-04 loss=2.5967 ce=2.5967 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.5943180490867235e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007349636871367693, 'last_eos_top1': 0} +step=214650 lr=6.000000e-04 loss=2.0961 ce=2.0961 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.648377398552839e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004934821045026183, 'last_eos_top1': 0} +step=214700 lr=6.000000e-04 loss=2.1363 ce=2.1363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6252899513347074e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0012421307619661093, 'last_eos_top1': 0} +step=214750 lr=6.000000e-04 loss=2.3887 ce=2.3887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.357755071599968e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000775542575865984, 'last_eos_top1': 0} +step=214800 lr=6.000000e-04 loss=2.2925 ce=2.2925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.212615284719504e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001038718968629837, 'last_eos_top1': 0} +step=214850 lr=6.000000e-04 loss=2.4708 ce=2.4708 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.882809536065906e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013781605521216989, 'last_eos_top1': 0} +step=214900 lr=6.000000e-04 loss=2.1284 ce=2.1284 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.751891613821499e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00039928738260641694, 'last_eos_top1': 0} +step=214950 lr=6.000000e-04 loss=2.8453 ce=2.8453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1114173503301572e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009557706071063876, 'last_eos_top1': 0} +step=215000 lr=6.000000e-04 loss=2.5567 ce=2.5567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.231703991943505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006103357300162315, 'last_eos_top1': 0} +step=215050 lr=6.000000e-04 loss=2.6243 ce=2.6243 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5135787988547236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005050242762081325, 'last_eos_top1': 0} +step=215100 lr=6.000000e-04 loss=2.4366 ce=2.4366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.779135219403543e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009643843513913453, 'last_eos_top1': 0} +step=215150 lr=6.000000e-04 loss=2.5702 ce=2.5702 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.157152827450773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011944083962589502, 'last_eos_top1': 0} +step=215200 lr=6.000000e-04 loss=2.4408 ce=2.4408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.906531441723928e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015399372205138206, 'last_eos_top1': 0} +step=215250 lr=6.000000e-04 loss=3.1587 ce=3.1587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.913902673957637e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011549204355105758, 'last_eos_top1': 0} +step=215300 lr=6.000000e-04 loss=2.5232 ce=2.5232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.113762083259644e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008353067096322775, 'last_eos_top1': 0} +step=215350 lr=6.000000e-04 loss=2.9332 ce=2.9332 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5164271139365155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006821817369200289, 'last_eos_top1': 0} +step=215400 lr=6.000000e-04 loss=2.6632 ce=2.6632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4181083467265125e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024578156881034374, 'last_eos_top1': 0} +step=215450 lr=6.000000e-04 loss=2.8974 ce=2.8974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.39478855393827e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029657017439603806, 'last_eos_top1': 0} +step=215500 lr=6.000000e-04 loss=2.8821 ce=2.8821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2495014743763022e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034598070196807384, 'last_eos_top1': 0} +step=215550 lr=6.000000e-04 loss=2.7386 ce=2.7386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.344983714574482e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002971246838569641, 'last_eos_top1': 0} +step=215600 lr=6.000000e-04 loss=2.4199 ce=2.4199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.492732841754332e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002095852978527546, 'last_eos_top1': 0} +step=215650 lr=6.000000e-04 loss=2.7122 ce=2.7122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2581037885393016e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026843282394111156, 'last_eos_top1': 0} +step=215700 lr=6.000000e-04 loss=2.4947 ce=2.4947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.267825144983362e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002325505018234253, 'last_eos_top1': 0} +step=215750 lr=6.000000e-04 loss=2.6129 ce=2.6129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.401041097589768e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026817680336534977, 'last_eos_top1': 0} +step=215800 lr=6.000000e-04 loss=2.5968 ce=2.5968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3909120752941817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029709101654589176, 'last_eos_top1': 0} +step=215850 lr=6.000000e-04 loss=2.3821 ce=2.3821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2667461305682082e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002884605899453163, 'last_eos_top1': 0} +step=215900 lr=6.000000e-04 loss=2.6417 ce=2.6417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.802622687653638e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026154338847845793, 'last_eos_top1': 0} +step=215950 lr=6.000000e-04 loss=2.5854 ce=2.5854 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0433035640744492e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025306243915110826, 'last_eos_top1': 0} +step=216000 lr=6.000000e-04 loss=2.4939 ce=2.4939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1317269127175678e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00187777413520962, 'last_eos_top1': 0} +step=216050 lr=6.000000e-04 loss=2.7527 ce=2.7527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.805140401411336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00143746891990304, 'last_eos_top1': 0} +step=216100 lr=6.000000e-04 loss=2.3600 ce=2.3600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6242379287141375e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006219361675903201, 'last_eos_top1': 0} +step=216150 lr=6.000000e-04 loss=2.7111 ce=2.7111 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.403654879250098e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000911691808141768, 'last_eos_top1': 0} +step=216200 lr=6.000000e-04 loss=3.0732 ce=3.0732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.874287358485162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0010668212780728936, 'last_eos_top1': 0} +step=216250 lr=6.000000e-04 loss=2.1910 ce=2.1910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.248529876349494e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018378505483269691, 'last_eos_top1': 0} +step=216300 lr=6.000000e-04 loss=3.0024 ce=3.0024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8864426389627624e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002946040127426386, 'last_eos_top1': 0} +step=216350 lr=6.000000e-04 loss=2.4327 ce=2.4327 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.344213604985271e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004063974134624004, 'last_eos_top1': 0} +step=216400 lr=6.000000e-04 loss=2.4736 ce=2.4736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.314436177490279e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003865897888317704, 'last_eos_top1': 0} +step=216450 lr=6.000000e-04 loss=2.1499 ce=2.1499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3956424633506685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030042966827750206, 'last_eos_top1': 0} +step=216500 lr=6.000000e-04 loss=2.5420 ce=2.5420 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.245348908531014e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002350473776459694, 'last_eos_top1': 0} +step=216550 lr=6.000000e-04 loss=2.2423 ce=2.2423 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.004509941703873e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026844548992812634, 'last_eos_top1': 0} +step=216600 lr=6.000000e-04 loss=2.5286 ce=2.5286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1589954738155939e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001748042763210833, 'last_eos_top1': 0} +step=216650 lr=6.000000e-04 loss=2.9381 ce=2.9381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.87860563932918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002710338681936264, 'last_eos_top1': 0} +step=216700 lr=6.000000e-04 loss=2.7426 ce=2.7426 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.280767931661103e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017350094858556986, 'last_eos_top1': 0} +step=216750 lr=6.000000e-04 loss=3.0509 ce=3.0509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.179330168582965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023162802681326866, 'last_eos_top1': 0} +step=216800 lr=6.000000e-04 loss=2.6237 ce=2.6237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2800850527128205e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036017578095197678, 'last_eos_top1': 0} +step=216850 lr=6.000000e-04 loss=2.9655 ce=2.9655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.768640327616595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038833755534142256, 'last_eos_top1': 0} +step=216900 lr=6.000000e-04 loss=2.9424 ce=2.9424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.389190275513101e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004206488840281963, 'last_eos_top1': 0} +step=216950 lr=6.000000e-04 loss=2.3890 ce=2.3890 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.77369406123762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00395531952381134, 'last_eos_top1': 0} +step=217000 lr=6.000000e-04 loss=2.6859 ce=2.6859 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.543008112203097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035668802447617054, 'last_eos_top1': 0} +step=217050 lr=6.000000e-04 loss=2.4773 ce=2.4773 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6014251741580665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004069505259394646, 'last_eos_top1': 0} +step=217100 lr=6.000000e-04 loss=2.8511 ce=2.8511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2406718926504254e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005303464829921722, 'last_eos_top1': 0} +step=217150 lr=6.000000e-04 loss=2.3118 ce=2.3118 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2932870201184414e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004780531860888004, 'last_eos_top1': 0} +step=217200 lr=6.000000e-04 loss=2.6841 ce=2.6841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0362376997363754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004866686649620533, 'last_eos_top1': 0} +step=217250 lr=6.000000e-04 loss=2.1772 ce=2.1772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9089369743596762e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003784253727644682, 'last_eos_top1': 0} +step=217300 lr=6.000000e-04 loss=2.3138 ce=2.3138 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.079477993305773e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003419904038310051, 'last_eos_top1': 0} +step=217350 lr=6.000000e-04 loss=2.4730 ce=2.4730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0732489499787334e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002550013829022646, 'last_eos_top1': 0} +step=217400 lr=6.000000e-04 loss=1.8770 ce=1.8770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.421946404268965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00207491684705019, 'last_eos_top1': 0} +step=217450 lr=6.000000e-04 loss=2.0003 ce=2.0003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.088305104232859e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002411765046417713, 'last_eos_top1': 0} +step=217500 lr=6.000000e-04 loss=2.7776 ce=2.7776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0545907065970823e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001373046776279807, 'last_eos_top1': 0} +step=217550 lr=6.000000e-04 loss=2.6681 ce=2.6681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.067701411258895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026586854364722967, 'last_eos_top1': 0} +step=217600 lr=6.000000e-04 loss=2.7515 ce=2.7515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.740070264437236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032602837309241295, 'last_eos_top1': 0} +step=217650 lr=6.000000e-04 loss=2.1186 ce=2.1186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.952170901466161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026861154474318027, 'last_eos_top1': 0} +step=217700 lr=6.000000e-04 loss=2.0088 ce=2.0088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.359050530591048e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032534883357584476, 'last_eos_top1': 0} +step=217750 lr=6.000000e-04 loss=2.4225 ce=2.4225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.505136975145433e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038516558706760406, 'last_eos_top1': 0} +step=217800 lr=6.000000e-04 loss=2.3838 ce=2.3838 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6322769523831084e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004295914433896542, 'last_eos_top1': 0} +step=217850 lr=6.000000e-04 loss=2.2031 ce=2.2031 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.545179869339336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004707354120910168, 'last_eos_top1': 0} +step=217900 lr=6.000000e-04 loss=2.6570 ce=2.6570 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.466141480312217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004621037282049656, 'last_eos_top1': 0} +step=217950 lr=6.000000e-04 loss=2.6277 ce=2.6277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.388005436339881e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00399761600419879, 'last_eos_top1': 0} +step=218000 lr=6.000000e-04 loss=2.8551 ce=2.8551 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.34173307439778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005790287628769875, 'last_eos_top1': 0} +step=218050 lr=6.000000e-04 loss=2.5310 ce=2.5310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2189929040905554e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044132680632174015, 'last_eos_top1': 0} +step=218100 lr=6.000000e-04 loss=2.2948 ce=2.2948 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.660771866037976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005850737914443016, 'last_eos_top1': 0} +step=218150 lr=6.000000e-04 loss=2.6516 ce=2.6516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4768937035114504e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003932323306798935, 'last_eos_top1': 0} +step=218200 lr=6.000000e-04 loss=2.3633 ce=2.3633 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.650310519442428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033714016899466515, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=7 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3500000 rows=3491809 buffered=8191 part=openwebtext-t5 +step=218250 lr=6.000000e-04 loss=2.4682 ce=2.4682 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.820542800298426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004079599864780903, 'last_eos_top1': 0} +step=218300 lr=6.000000e-04 loss=2.3509 ce=2.3509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.266966465773294e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034015150740742683, 'last_eos_top1': 0} +step=218350 lr=6.000000e-04 loss=3.0738 ce=3.0738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.057640348444693e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00281510129570961, 'last_eos_top1': 0} +step=218400 lr=6.000000e-04 loss=2.2830 ce=2.2830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.276814590615686e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003108799457550049, 'last_eos_top1': 0} +step=218450 lr=6.000000e-04 loss=2.5098 ce=2.5098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2808945939468686e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004193063825368881, 'last_eos_top1': 0} +step=218500 lr=6.000000e-04 loss=3.0205 ce=3.0205 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1189735232619569e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004341744817793369, 'last_eos_top1': 0} +step=218550 lr=6.000000e-04 loss=2.1040 ce=2.1040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.438710675458424e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036201728507876396, 'last_eos_top1': 0} +step=218600 lr=6.000000e-04 loss=2.5807 ce=2.5807 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.263549271854572e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003992652986198664, 'last_eos_top1': 0} +step=218650 lr=6.000000e-04 loss=2.4404 ce=2.4404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.016839845164213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054167406633496284, 'last_eos_top1': 0} +step=218700 lr=6.000000e-04 loss=2.6136 ce=2.6136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.374024451361038e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036548241041600704, 'last_eos_top1': 0} +step=218750 lr=6.000000e-04 loss=2.6211 ce=2.6211 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.999940862646326e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00593812670558691, 'last_eos_top1': 0} +step=218800 lr=6.000000e-04 loss=2.1233 ce=2.1233 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.962166258337675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0045793624594807625, 'last_eos_top1': 0} +step=218850 lr=6.000000e-04 loss=2.9206 ce=2.9206 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0371338551922236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003997415769845247, 'last_eos_top1': 0} +step=218900 lr=6.000000e-04 loss=2.4360 ce=2.4360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.97911525296513e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004709969274699688, 'last_eos_top1': 0} +step=218950 lr=6.000000e-04 loss=2.6746 ce=2.6746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7555077647557482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004731061868369579, 'last_eos_top1': 0} +step=219000 lr=6.000000e-04 loss=2.4425 ce=2.4425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.053942833095789e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00566763523966074, 'last_eos_top1': 0} +step=219050 lr=6.000000e-04 loss=2.4289 ce=2.4289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3755397958448157e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0053099459037184715, 'last_eos_top1': 0} +step=219100 lr=6.000000e-04 loss=3.1016 ce=3.1016 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4608689525630325e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005559045821428299, 'last_eos_top1': 0} +step=219150 lr=6.000000e-04 loss=2.3766 ce=2.3766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.632819688296877e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005271849222481251, 'last_eos_top1': 0} +step=219200 lr=6.000000e-04 loss=2.9845 ce=2.9845 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3696719482541084e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005955856759101152, 'last_eos_top1': 0} +step=219250 lr=6.000000e-04 loss=2.3045 ce=2.3045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.85698284744285e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0055717290379107, 'last_eos_top1': 0} +step=219300 lr=6.000000e-04 loss=1.8869 ce=1.8869 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.944931323640049e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006248241290450096, 'last_eos_top1': 0} +step=219350 lr=6.000000e-04 loss=2.0882 ce=2.0882 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.791863259219099e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0047053927555680275, 'last_eos_top1': 0} +step=219400 lr=6.000000e-04 loss=2.7844 ce=2.7844 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6674330254318193e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004401133395731449, 'last_eos_top1': 0} +step=219450 lr=6.000000e-04 loss=2.7463 ce=2.7463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8860804630094208e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037477239966392517, 'last_eos_top1': 0} +step=219500 lr=6.000000e-04 loss=2.7912 ce=2.7912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4207678759703413e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002891494892537594, 'last_eos_top1': 0} +step=219550 lr=6.000000e-04 loss=2.8319 ce=2.8319 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4108441064308863e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030844886787235737, 'last_eos_top1': 0} +step=219600 lr=6.000000e-04 loss=2.4035 ce=2.4035 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1679845556500368e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033911559730768204, 'last_eos_top1': 0} +step=219650 lr=6.000000e-04 loss=2.5699 ce=2.5699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.308395990752615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003010394051671028, 'last_eos_top1': 0} +step=219700 lr=6.000000e-04 loss=2.4328 ce=2.4328 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6828081394778565e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034554293379187584, 'last_eos_top1': 0} +step=219750 lr=6.000000e-04 loss=2.4693 ce=2.4693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3314258467289619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004159300588071346, 'last_eos_top1': 0} +step=219800 lr=6.000000e-04 loss=2.3816 ce=2.3816 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.401694630156271e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035819655749946833, 'last_eos_top1': 0} +step=219850 lr=6.000000e-04 loss=2.7408 ce=2.7408 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.264191026479239e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036056716926395893, 'last_eos_top1': 0} +step=219900 lr=6.000000e-04 loss=2.4526 ce=2.4526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.242039257311262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030954028479754925, 'last_eos_top1': 0} +step=219950 lr=6.000000e-04 loss=2.3096 ce=2.3096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3806071365252137e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00374825787730515, 'last_eos_top1': 0} +step=220000 lr=6.000000e-04 loss=2.3407 ce=2.3407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8641021597431973e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004446822218596935, 'last_eos_top1': 0} +step=220050 lr=6.000000e-04 loss=2.7044 ce=2.7044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.63256554515101e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004257007036358118, 'last_eos_top1': 0} +step=220100 lr=6.000000e-04 loss=2.6086 ce=2.6086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.52844334783731e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003965741489082575, 'last_eos_top1': 0} +step=220150 lr=6.000000e-04 loss=2.9341 ce=2.9341 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.793069350242149e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005552203394472599, 'last_eos_top1': 0} +step=220200 lr=6.000000e-04 loss=2.6087 ce=2.6087 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.706278999336064e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004815526306629181, 'last_eos_top1': 0} +step=220250 lr=6.000000e-04 loss=2.5517 ce=2.5517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5676949260523543e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0046672336757183075, 'last_eos_top1': 0} +step=220300 lr=6.000000e-04 loss=2.7041 ce=2.7041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.891700650157873e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005855767987668514, 'last_eos_top1': 0} +step=220350 lr=6.000000e-04 loss=3.0253 ce=3.0253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.673324034840334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004728122614324093, 'last_eos_top1': 0} +step=220400 lr=6.000000e-04 loss=2.4652 ce=2.4652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.284631465154234e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036067343316972256, 'last_eos_top1': 0} +step=220450 lr=6.000000e-04 loss=2.3002 ce=2.3002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.18410388394841e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005938605405390263, 'last_eos_top1': 0} +step=220500 lr=6.000000e-04 loss=2.1768 ce=2.1768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8331294995732605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065517909824848175, 'last_eos_top1': 0} +step=220550 lr=6.000000e-04 loss=2.2137 ce=2.2137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.289066221943358e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006112420931458473, 'last_eos_top1': 0} +step=220600 lr=6.000000e-04 loss=2.4072 ce=2.4072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4208766010124236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0047783199697732925, 'last_eos_top1': 0} +step=220650 lr=6.000000e-04 loss=2.6154 ce=2.6154 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.838214514777064e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004079536069184542, 'last_eos_top1': 0} +step=220700 lr=6.000000e-04 loss=2.5171 ce=2.5171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3074060916551389e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037672456819564104, 'last_eos_top1': 0} +step=220750 lr=6.000000e-04 loss=2.2836 ce=2.2836 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.483099696197314e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023861376103013754, 'last_eos_top1': 0} +step=220800 lr=6.000000e-04 loss=2.6365 ce=2.6365 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4296813787950668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035216109827160835, 'last_eos_top1': 0} +step=220850 lr=6.000000e-04 loss=2.1279 ce=2.1279 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.260014975443482e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004901241976767778, 'last_eos_top1': 0} +step=220900 lr=6.000000e-04 loss=2.2366 ce=2.2366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.645876884344034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054564094170928, 'last_eos_top1': 0} +step=220950 lr=6.000000e-04 loss=2.5379 ce=2.5379 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7228123397217132e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006521584931761026, 'last_eos_top1': 0} +step=221000 lr=6.000000e-04 loss=2.7685 ce=2.7685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5329535421624314e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034607560373842716, 'last_eos_top1': 0} +step=221050 lr=6.000000e-04 loss=2.8371 ce=2.8371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.439381195377791e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005124721676111221, 'last_eos_top1': 0} +step=221100 lr=6.000000e-04 loss=2.5626 ce=2.5626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3839688108419068e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004702150821685791, 'last_eos_top1': 0} +step=221150 lr=6.000000e-04 loss=2.2045 ce=2.2045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.684871550532989e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038845306262373924, 'last_eos_top1': 0} +step=221200 lr=6.000000e-04 loss=2.4962 ce=2.4962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.956475171027705e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005034309811890125, 'last_eos_top1': 0} +step=221250 lr=6.000000e-04 loss=2.9090 ce=2.9090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1414374714368023e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005551867187023163, 'last_eos_top1': 0} +step=221300 lr=6.000000e-04 loss=1.9061 ce=1.9061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.50116302393144e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005283638369292021, 'last_eos_top1': 0} +step=221350 lr=6.000000e-04 loss=1.6510 ce=1.6510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.487815310014412e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00462024100124836, 'last_eos_top1': 0} +step=221400 lr=6.000000e-04 loss=2.5689 ce=2.5689 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9714705558726564e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007129417732357979, 'last_eos_top1': 0} +step=221450 lr=6.000000e-04 loss=2.4513 ce=2.4513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.715398507774808e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005094136111438274, 'last_eos_top1': 0} +step=221500 lr=6.000000e-04 loss=2.4833 ce=2.4833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.508189249463612e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005501905456185341, 'last_eos_top1': 0} +step=221550 lr=6.000000e-04 loss=2.6845 ce=2.6845 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.842117388965562e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003934197127819061, 'last_eos_top1': 0} +step=221600 lr=6.000000e-04 loss=2.5095 ce=2.5095 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.304719707055483e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004616663791239262, 'last_eos_top1': 0} +step=221650 lr=6.000000e-04 loss=2.5392 ce=2.5392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.123802849178901e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0056658340618014336, 'last_eos_top1': 0} +step=221700 lr=6.000000e-04 loss=2.5816 ce=2.5816 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3188240447780117e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004104963503777981, 'last_eos_top1': 0} +step=221750 lr=6.000000e-04 loss=2.8561 ce=2.8561 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.485907190130092e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003609014442190528, 'last_eos_top1': 0} +step=221800 lr=6.000000e-04 loss=2.4609 ce=2.4609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.09343032920151e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035334862768650055, 'last_eos_top1': 0} +step=221850 lr=6.000000e-04 loss=2.2780 ce=2.2780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1037918739020824e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003855567192658782, 'last_eos_top1': 0} +step=221900 lr=6.000000e-04 loss=2.4780 ce=2.4780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.615670150087681e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003026300109922886, 'last_eos_top1': 0} +step=221950 lr=6.000000e-04 loss=2.8483 ce=2.8483 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5869583876337856e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006206332705914974, 'last_eos_top1': 0} +step=222000 lr=6.000000e-04 loss=2.7613 ce=2.7613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.077950729377335e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033074100501835346, 'last_eos_top1': 0} +step=222050 lr=6.000000e-04 loss=2.1465 ce=2.1465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4188295608619228e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006368532776832581, 'last_eos_top1': 0} +step=222100 lr=6.000000e-04 loss=2.3877 ce=2.3877 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.058212001225911e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010046223178505898, 'last_eos_top1': 0} +step=222150 lr=6.000000e-04 loss=2.5755 ce=2.5755 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9271876226412132e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011655354872345924, 'last_eos_top1': 0} +step=222200 lr=6.000000e-04 loss=2.7281 ce=2.7281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.011078873299994e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007199062965810299, 'last_eos_top1': 0} +step=222250 lr=6.000000e-04 loss=2.5359 ce=2.5359 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.318840183259454e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005114064551889896, 'last_eos_top1': 0} +step=222300 lr=6.000000e-04 loss=2.1324 ce=2.1324 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.540326133588678e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0068208277225494385, 'last_eos_top1': 0} +step=222350 lr=6.000000e-04 loss=2.4525 ce=2.4525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.586544936406426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005409521050751209, 'last_eos_top1': 0} +step=222400 lr=6.000000e-04 loss=2.2873 ce=2.2873 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.360322913678829e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007591228932142258, 'last_eos_top1': 0} +step=222450 lr=6.000000e-04 loss=2.4344 ce=2.4344 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4041548638488166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008828149177134037, 'last_eos_top1': 0} +step=222500 lr=6.000000e-04 loss=2.7114 ce=2.7114 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.825696462125052e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011417379602789879, 'last_eos_top1': 0} +step=222550 lr=6.000000e-04 loss=2.3709 ce=2.3709 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.914415386447217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009312320500612259, 'last_eos_top1': 0} +step=222600 lr=6.000000e-04 loss=2.3482 ce=2.3482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.514550139196217e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0095272958278656, 'last_eos_top1': 0} +step=222650 lr=6.000000e-04 loss=2.2871 ce=2.2871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1935244149062783e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006604418158531189, 'last_eos_top1': 0} +step=222700 lr=6.000000e-04 loss=2.5067 ce=2.5067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2813420653401408e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007658612448722124, 'last_eos_top1': 0} +step=222750 lr=6.000000e-04 loss=2.4685 ce=2.4685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0357251994719263e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008204981684684753, 'last_eos_top1': 0} +step=222800 lr=6.000000e-04 loss=2.7211 ce=2.7211 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9747290934901685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008012712933123112, 'last_eos_top1': 0} +step=222850 lr=6.000000e-04 loss=2.4509 ce=2.4509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6463713109260425e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007700217422097921, 'last_eos_top1': 0} +step=222900 lr=6.000000e-04 loss=3.0263 ce=3.0263 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.98353744862834e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005703897215425968, 'last_eos_top1': 0} +step=222950 lr=6.000000e-04 loss=2.7748 ce=2.7748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2683417379739694e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005889147985726595, 'last_eos_top1': 0} +step=223000 lr=6.000000e-04 loss=2.8402 ce=2.8402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.859374818508513e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005582061130553484, 'last_eos_top1': 0} +step=223050 lr=6.000000e-04 loss=2.3561 ce=2.3561 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.528326361556537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0059670922346413136, 'last_eos_top1': 0} +step=223100 lr=6.000000e-04 loss=2.6634 ce=2.6634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.032076387578854e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00611452292650938, 'last_eos_top1': 0} +step=223150 lr=6.000000e-04 loss=2.4879 ce=2.4879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.949611921911128e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006461211480200291, 'last_eos_top1': 0} +step=223200 lr=6.000000e-04 loss=2.6876 ce=2.6876 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.192781564895995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006021053530275822, 'last_eos_top1': 0} +step=223250 lr=6.000000e-04 loss=1.9947 ce=1.9947 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.029396783764241e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005446199327707291, 'last_eos_top1': 0} +step=223300 lr=6.000000e-04 loss=2.3012 ce=2.3012 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0464535080245696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005133269354701042, 'last_eos_top1': 0} +step=223350 lr=6.000000e-04 loss=2.4486 ce=2.4486 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.222883141366765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005770857445895672, 'last_eos_top1': 0} +step=223400 lr=6.000000e-04 loss=2.0288 ce=2.0288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1678132472734433e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005050447769463062, 'last_eos_top1': 0} +step=223450 lr=6.000000e-04 loss=2.2999 ce=2.2999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3274591765366495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0046272058971226215, 'last_eos_top1': 0} +step=223500 lr=6.000000e-04 loss=2.5957 ce=2.5957 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0549105354584754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007044234313070774, 'last_eos_top1': 0} +step=223550 lr=6.000000e-04 loss=2.4033 ce=2.4033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.887164196697995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006969451904296875, 'last_eos_top1': 0} +step=223600 lr=6.000000e-04 loss=2.1194 ce=2.1194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.280153113242704e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006349899340420961, 'last_eos_top1': 0} +step=223650 lr=6.000000e-04 loss=2.1782 ce=2.1782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.874295417626854e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007961070165038109, 'last_eos_top1': 0} +step=223700 lr=6.000000e-04 loss=2.3333 ce=2.3333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0186908184550703e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007333950139582157, 'last_eos_top1': 0} +step=223750 lr=6.000000e-04 loss=2.6067 ce=2.6067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.575757429847727e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007345516234636307, 'last_eos_top1': 0} +step=223800 lr=6.000000e-04 loss=2.3699 ce=2.3699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1077316230512224e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007345446385443211, 'last_eos_top1': 0} +step=223850 lr=6.000000e-04 loss=2.6024 ce=2.6024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.24585482000839e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007410856895148754, 'last_eos_top1': 0} +step=223900 lr=6.000000e-04 loss=3.0554 ce=3.0554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.590379704604857e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006224261596798897, 'last_eos_top1': 0} +step=223950 lr=6.000000e-04 loss=2.5055 ce=2.5055 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7481761460658163e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065386490896344185, 'last_eos_top1': 0} +step=224000 lr=6.000000e-04 loss=2.5307 ce=2.5307 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7577060134499334e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008900346234440804, 'last_eos_top1': 0} +step=224050 lr=6.000000e-04 loss=2.4587 ce=2.4587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.486019628937356e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006950061768293381, 'last_eos_top1': 0} +step=224100 lr=6.000000e-04 loss=3.2505 ce=3.2505 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2468237400753424e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00707099586725235, 'last_eos_top1': 0} +step=224150 lr=6.000000e-04 loss=2.5378 ce=2.5378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.341747161466628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037899422459304333, 'last_eos_top1': 0} +step=224200 lr=6.000000e-04 loss=2.4742 ce=2.4742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1800411812146194e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005771174561232328, 'last_eos_top1': 0} +step=224250 lr=6.000000e-04 loss=3.4169 ce=3.4169 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.850777365092654e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006679621059447527, 'last_eos_top1': 0} +step=224300 lr=6.000000e-04 loss=2.5616 ce=2.5616 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.93438027560478e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0074881454929709435, 'last_eos_top1': 0} +step=224350 lr=6.000000e-04 loss=2.2217 ce=2.2217 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0963673048536293e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007286188192665577, 'last_eos_top1': 0} +step=224400 lr=6.000000e-04 loss=2.6651 ce=2.6651 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3710418796981685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00757858669385314, 'last_eos_top1': 0} +step=224450 lr=6.000000e-04 loss=2.0583 ce=2.0583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0579071386018768e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007836422882974148, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=3] docs=3600000 rows=3591809 buffered=8191 part=openwebtext-t5 + +step=224500 lr=6.000000e-04 loss=2.3593 ce=2.3593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2262486052350141e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007751096971333027, 'last_eos_top1': 0} +step=224550 lr=6.000000e-04 loss=2.6121 ce=2.6121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.151864792831475e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00657563004642725, 'last_eos_top1': 0} +step=224600 lr=6.000000e-04 loss=2.3951 ce=2.3951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1553796866792254e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006028598174452782, 'last_eos_top1': 0} +step=224650 lr=6.000000e-04 loss=1.9221 ce=1.9221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8078959328704514e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009223947301506996, 'last_eos_top1': 0} +step=224700 lr=6.000000e-04 loss=1.8564 ce=1.8564 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.723404218675569e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011072840541601181, 'last_eos_top1': 0} +step=224750 lr=6.000000e-04 loss=2.7950 ce=2.7950 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.318302540516015e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012467622756958008, 'last_eos_top1': 0} +step=224800 lr=6.000000e-04 loss=2.3673 ce=2.3673 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5609062756993808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009999990463256836, 'last_eos_top1': 0} +step=224850 lr=6.000000e-04 loss=2.7176 ce=2.7176 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.288837307190988e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011109311133623123, 'last_eos_top1': 0} +step=224900 lr=6.000000e-04 loss=2.1893 ce=2.1893 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6708563634892926e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011331592686474323, 'last_eos_top1': 0} +step=224950 lr=6.000000e-04 loss=2.5309 ce=2.5309 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4964197362132836e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010688901878893375, 'last_eos_top1': 0} +step=225000 lr=6.000000e-04 loss=2.5326 ce=2.5326 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.488500831532292e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009197024628520012, 'last_eos_top1': 0} +step=225050 lr=6.000000e-04 loss=2.4666 ce=2.4666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9746341422433034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006566227413713932, 'last_eos_top1': 0} +step=225100 lr=6.000000e-04 loss=1.9634 ce=1.9634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.745073394791689e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007483606692403555, 'last_eos_top1': 0} +step=225150 lr=6.000000e-04 loss=2.5298 ce=2.5298 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.720627086702734e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008269098587334156, 'last_eos_top1': 0} +step=225200 lr=6.000000e-04 loss=2.1185 ce=2.1185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.174715160042979e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009905191138386726, 'last_eos_top1': 0} +step=225250 lr=6.000000e-04 loss=2.6648 ce=2.6648 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3231363482191227e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009141460992395878, 'last_eos_top1': 0} +step=225300 lr=6.000000e-04 loss=2.5410 ce=2.5410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.267673309892416e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00842004083096981, 'last_eos_top1': 0} +step=225350 lr=6.000000e-04 loss=2.3476 ce=2.3476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8208917026640847e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0062460023909807205, 'last_eos_top1': 0} +step=225400 lr=6.000000e-04 loss=2.3402 ce=2.3402 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.271762539545307e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005144360475242138, 'last_eos_top1': 0} +step=225450 lr=6.000000e-04 loss=2.3267 ce=2.3267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2206301107653417e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0069264741614460945, 'last_eos_top1': 0} +step=225500 lr=6.000000e-04 loss=2.9074 ce=2.9074 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1142291441501584e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006837797351181507, 'last_eos_top1': 0} +step=225550 lr=6.000000e-04 loss=2.2260 ce=2.2260 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.641431693627965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006751334760338068, 'last_eos_top1': 0} +step=225600 lr=6.000000e-04 loss=2.5458 ce=2.5458 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5289051538711647e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006842130795121193, 'last_eos_top1': 0} +step=225650 lr=6.000000e-04 loss=1.8244 ce=1.8244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.788564628048334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006848976947367191, 'last_eos_top1': 0} +step=225700 lr=6.000000e-04 loss=2.9584 ce=2.9584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2264792935166042e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007426348980516195, 'last_eos_top1': 0} +step=225750 lr=6.000000e-04 loss=2.9929 ce=2.9929 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.043422333983472e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008425118401646614, 'last_eos_top1': 0} +step=225800 lr=6.000000e-04 loss=2.2681 ce=2.2681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.50171648178366e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008694179356098175, 'last_eos_top1': 0} +step=225850 lr=6.000000e-04 loss=2.6258 ce=2.6258 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.958692317013629e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009625546634197235, 'last_eos_top1': 0} +step=225900 lr=6.000000e-04 loss=2.5924 ce=2.5924 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.004746061691549e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007269898429512978, 'last_eos_top1': 0} +step=225950 lr=6.000000e-04 loss=3.0846 ce=3.0846 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.177559160074452e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006245927419513464, 'last_eos_top1': 0} +step=226000 lr=6.000000e-04 loss=2.8522 ce=2.8522 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1977956395130605e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006260890979319811, 'last_eos_top1': 0} +step=226050 lr=6.000000e-04 loss=2.4758 ce=2.4758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2720351151074283e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00572770182043314, 'last_eos_top1': 0} +step=226100 lr=6.000000e-04 loss=2.3147 ce=2.3147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.363025254860986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006284995470196009, 'last_eos_top1': 0} +step=226150 lr=6.000000e-04 loss=2.5454 ce=2.5454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.974619853077456e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005960883107036352, 'last_eos_top1': 0} +step=226200 lr=6.000000e-04 loss=2.5889 ce=2.5889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.177880201605149e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0042734090238809586, 'last_eos_top1': 0} +step=226250 lr=6.000000e-04 loss=2.0371 ce=2.0371 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.463171409151983e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004649344366043806, 'last_eos_top1': 0} +step=226300 lr=6.000000e-04 loss=2.5606 ce=2.5606 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4909675883245654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004710664972662926, 'last_eos_top1': 0} +step=226350 lr=6.000000e-04 loss=2.5532 ce=2.5532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.059990598965669e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005144293420016766, 'last_eos_top1': 0} +step=226400 lr=6.000000e-04 loss=2.3066 ce=2.3066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3368231520871632e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006725343409925699, 'last_eos_top1': 0} +step=226450 lr=6.000000e-04 loss=2.7736 ce=2.7736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.7334231061977334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006077935453504324, 'last_eos_top1': 0} +step=226500 lr=6.000000e-04 loss=2.4532 ce=2.4532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.553842467837967e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006048933602869511, 'last_eos_top1': 0} +step=226550 lr=6.000000e-04 loss=2.3823 ce=2.3823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1037936878419714e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005720125045627356, 'last_eos_top1': 0} +step=226600 lr=6.000000e-04 loss=2.7282 ce=2.7282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.60720548289828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00426526740193367, 'last_eos_top1': 0} +step=226650 lr=6.000000e-04 loss=2.6927 ce=2.6927 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7579954955144785e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00525871105492115, 'last_eos_top1': 0} +step=226700 lr=6.000000e-04 loss=1.9889 ce=1.9889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0082214834692422e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004721188917756081, 'last_eos_top1': 0} +step=226750 lr=6.000000e-04 loss=2.5446 ce=2.5446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.192847998434445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004353753291070461, 'last_eos_top1': 0} +step=226800 lr=6.000000e-04 loss=1.9615 ce=1.9615 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0384123824187554e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005273065529763699, 'last_eos_top1': 0} +step=226850 lr=6.000000e-04 loss=3.1422 ce=3.1422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.830950506904628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029254290275275707, 'last_eos_top1': 0} +step=226900 lr=6.000000e-04 loss=2.5143 ce=2.5143 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5061735868803225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035949009470641613, 'last_eos_top1': 0} +step=226950 lr=6.000000e-04 loss=2.3938 ce=2.3938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.958312506962102e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033917012624442577, 'last_eos_top1': 0} +step=227000 lr=6.000000e-04 loss=2.3231 ce=2.3231 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6466372017021058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003679482266306877, 'last_eos_top1': 0} +step=227050 lr=6.000000e-04 loss=2.5613 ce=2.5613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5210687201470137e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003546313149854541, 'last_eos_top1': 0} +step=227100 lr=6.000000e-04 loss=2.2046 ce=2.2046 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.795724857511232e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004641501232981682, 'last_eos_top1': 0} +step=227150 lr=6.000000e-04 loss=2.4853 ce=2.4853 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2914272701891605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005381675437092781, 'last_eos_top1': 0} +step=227200 lr=6.000000e-04 loss=1.9924 ce=1.9924 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0901088899117894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005417939275503159, 'last_eos_top1': 0} +step=227250 lr=6.000000e-04 loss=3.0998 ce=3.0998 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1416464076319244e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004296580795198679, 'last_eos_top1': 0} +step=227300 lr=6.000000e-04 loss=2.4818 ce=2.4818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.576878720603418e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00400170125067234, 'last_eos_top1': 0} +step=227350 lr=6.000000e-04 loss=2.9282 ce=2.9282 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0172314432566054e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005190784111618996, 'last_eos_top1': 0} +step=227400 lr=6.000000e-04 loss=2.5232 ce=2.5232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1756735148082953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004224271513521671, 'last_eos_top1': 0} +step=227450 lr=6.000000e-04 loss=2.3232 ce=2.3232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.485787475365214e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031160551588982344, 'last_eos_top1': 0} +step=227500 lr=6.000000e-04 loss=2.4509 ce=2.4509 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.320840268221218e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00306533370167017, 'last_eos_top1': 0} +step=227550 lr=6.000000e-04 loss=3.0863 ce=3.0863 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.756503691780381e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003653123741969466, 'last_eos_top1': 0} +step=227600 lr=6.000000e-04 loss=2.1137 ce=2.1137 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.2241987254528794e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004380323924124241, 'last_eos_top1': 0} +step=227650 lr=6.000000e-04 loss=2.9029 ce=2.9029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.63276488619158e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005460937507450581, 'last_eos_top1': 0} +step=227700 lr=6.000000e-04 loss=2.6308 ce=2.6308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9403265469009057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037568570114672184, 'last_eos_top1': 0} +step=227750 lr=6.000000e-04 loss=2.4462 ce=2.4462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4864370314171538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021355836652219296, 'last_eos_top1': 0} +step=227800 lr=6.000000e-04 loss=2.5654 ce=2.5654 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3437907000479754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019438675371930003, 'last_eos_top1': 0} +step=227850 lr=6.000000e-04 loss=2.8526 ce=2.8526 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0603795999486465e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021225004456937313, 'last_eos_top1': 0} +step=227900 lr=6.000000e-04 loss=2.8972 ce=2.8972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9447603335720487e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029429025016725063, 'last_eos_top1': 0} +step=227950 lr=6.000000e-04 loss=2.2584 ce=2.2584 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.898971256916411e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030405123252421618, 'last_eos_top1': 0} +step=228000 lr=6.000000e-04 loss=2.5241 ce=2.5241 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.691319529432803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034706545993685722, 'last_eos_top1': 0} +step=228050 lr=6.000000e-04 loss=2.8683 ce=2.8683 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.417851945006987e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00359442550688982, 'last_eos_top1': 0} +step=228100 lr=6.000000e-04 loss=2.3952 ce=2.3952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8877278711879626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029032942838966846, 'last_eos_top1': 0} +step=228150 lr=6.000000e-04 loss=2.4632 ce=2.4632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0002711860579439e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014356231549754739, 'last_eos_top1': 0} +step=228200 lr=6.000000e-04 loss=2.2496 ce=2.2496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8123132526379777e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016285553574562073, 'last_eos_top1': 0} +step=228250 lr=6.000000e-04 loss=2.0532 ce=2.0532 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.009452135913307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019022079650312662, 'last_eos_top1': 0} +step=228300 lr=6.000000e-04 loss=2.5718 ce=2.5718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.587201930116862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030625173822045326, 'last_eos_top1': 0} +step=228350 lr=6.000000e-04 loss=2.6425 ce=2.6425 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.554602749820333e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036576546262949705, 'last_eos_top1': 0} +step=228400 lr=6.000000e-04 loss=2.4884 ce=2.4884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.192686669237446e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030902246944606304, 'last_eos_top1': 0} +step=228450 lr=6.000000e-04 loss=2.8316 ce=2.8316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.289349923667032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003629765473306179, 'last_eos_top1': 0} +step=228500 lr=6.000000e-04 loss=2.7791 ce=2.7791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.823869054846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003201497718691826, 'last_eos_top1': 0} +step=228550 lr=6.000000e-04 loss=3.1513 ce=3.1513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.55385452369228e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004243599250912666, 'last_eos_top1': 0} +step=228600 lr=6.000000e-04 loss=2.8421 ce=2.8421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2852630788984243e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003983950242400169, 'last_eos_top1': 0} +step=228650 lr=6.000000e-04 loss=2.6445 ce=2.6445 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.57245504448656e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004402928054332733, 'last_eos_top1': 0} +step=228700 lr=6.000000e-04 loss=2.2982 ce=2.2982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.29075202596141e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003881350625306368, 'last_eos_top1': 0} +step=228750 lr=6.000000e-04 loss=2.4076 ce=2.4076 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0511049519409426e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028421252500265837, 'last_eos_top1': 0} +step=228800 lr=6.000000e-04 loss=2.2875 ce=2.2875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.491486601589713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029339788015931845, 'last_eos_top1': 0} +step=228850 lr=6.000000e-04 loss=2.6058 ce=2.6058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.410231773974374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026276232674717903, 'last_eos_top1': 0} +step=228900 lr=6.000000e-04 loss=1.8514 ce=1.8514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8717953430023044e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002267420059069991, 'last_eos_top1': 0} +step=228950 lr=6.000000e-04 loss=3.1903 ce=3.1903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.360780495626386e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002198190428316593, 'last_eos_top1': 0} +step=229000 lr=6.000000e-04 loss=2.2516 ce=2.2516 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.944671748788096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002065399195998907, 'last_eos_top1': 0} +step=229050 lr=6.000000e-04 loss=2.7762 ce=2.7762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7397200028644875e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003359937109053135, 'last_eos_top1': 0} +step=229100 lr=6.000000e-04 loss=2.3219 ce=2.3219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.210762876435183e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002589094452559948, 'last_eos_top1': 0} +step=229150 lr=6.000000e-04 loss=2.5459 ce=2.5459 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2844942830270156e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033845214638859034, 'last_eos_top1': 0} +step=229200 lr=6.000000e-04 loss=2.1735 ce=2.1735 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.665501809446141e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026479437947273254, 'last_eos_top1': 0} +step=229250 lr=6.000000e-04 loss=2.1171 ce=2.1171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.015858470054809e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034750900231301785, 'last_eos_top1': 0} +step=229300 lr=6.000000e-04 loss=2.3529 ce=2.3529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.77994137024507e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003303898498415947, 'last_eos_top1': 0} +step=229350 lr=6.000000e-04 loss=2.7098 ce=2.7098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.949695944058476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003343244083225727, 'last_eos_top1': 0} +step=229400 lr=6.000000e-04 loss=2.7823 ce=2.7823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.973980922950432e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034379479475319386, 'last_eos_top1': 0} +step=229450 lr=6.000000e-04 loss=2.0421 ce=2.0421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1111055755463894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037569806445389986, 'last_eos_top1': 0} +step=229500 lr=6.000000e-04 loss=2.4728 ce=2.4728 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.320791155507322e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037916721776127815, 'last_eos_top1': 0} +step=229550 lr=6.000000e-04 loss=2.4116 ce=2.4116 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.979383321478963e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004193508997559547, 'last_eos_top1': 0} +step=229600 lr=6.000000e-04 loss=2.9321 ce=2.9321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0654051948222332e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0046698544174432755, 'last_eos_top1': 0} +step=229650 lr=6.000000e-04 loss=2.7700 ce=2.7700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7379610653733835e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054601626470685005, 'last_eos_top1': 0} +step=229700 lr=6.000000e-04 loss=2.3681 ce=2.3681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4371806173585355e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006106057204306126, 'last_eos_top1': 0} +step=229750 lr=6.000000e-04 loss=3.2919 ce=3.2919 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0070814571226947e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004761627875268459, 'last_eos_top1': 0} +step=229800 lr=6.000000e-04 loss=2.8240 ce=2.8240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.243811291526072e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036217737942934036, 'last_eos_top1': 0} +step=229850 lr=6.000000e-04 loss=2.7227 ce=2.7227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.480328688165173e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034993914887309074, 'last_eos_top1': 0} +step=229900 lr=6.000000e-04 loss=2.5206 ce=2.5206 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.610458745446522e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003299703588709235, 'last_eos_top1': 0} +step=229950 lr=6.000000e-04 loss=2.6991 ce=2.6991 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.476935363432858e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004183365032076836, 'last_eos_top1': 0} +step=230000 lr=6.000000e-04 loss=2.7036 ce=2.7036 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0153733203187585e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003941217437386513, 'last_eos_top1': 0} +step=230050 lr=6.000000e-04 loss=2.6925 ce=2.6925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8565060599939898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004308777861297131, 'last_eos_top1': 0} +step=230100 lr=6.000000e-04 loss=2.4499 ce=2.4499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4152688891044818e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003722973633557558, 'last_eos_top1': 0} +step=230150 lr=6.000000e-04 loss=2.0119 ce=2.0119 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4181619917508215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00424556341022253, 'last_eos_top1': 0} +step=230200 lr=6.000000e-04 loss=2.2698 ce=2.2698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.101098289131187e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002771585714071989, 'last_eos_top1': 0} +step=230250 lr=6.000000e-04 loss=2.5380 ce=2.5380 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.728410880896263e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020914780907332897, 'last_eos_top1': 0} +step=230300 lr=6.000000e-04 loss=2.9192 ce=2.9192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.872312148218043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002541847061365843, 'last_eos_top1': 0} +step=230350 lr=6.000000e-04 loss=2.9067 ce=2.9067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.333180863293819e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032006572000682354, 'last_eos_top1': 0} +step=230400 lr=6.000000e-04 loss=2.4860 ce=2.4860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.567512663314119e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003693251870572567, 'last_eos_top1': 0} +step=230450 lr=6.000000e-04 loss=2.8322 ce=2.8322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0861776900128461e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0039338842034339905, 'last_eos_top1': 0} +step=230500 lr=6.000000e-04 loss=2.6753 ce=2.6753 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.32204147829907e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002533753402531147, 'last_eos_top1': 0} +step=230550 lr=6.000000e-04 loss=3.0262 ce=3.0262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.8004814996384084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033341660164296627, 'last_eos_top1': 0} +step=230600 lr=6.000000e-04 loss=3.1081 ce=3.1081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5482810340472497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026938756927847862, 'last_eos_top1': 0} +step=230650 lr=6.000000e-04 loss=2.3738 ce=2.3738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6338737370679155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003989310935139656, 'last_eos_top1': 0} +step=230700 lr=6.000000e-04 loss=2.8245 ce=2.8245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8559221643954515e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003965775948017836, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3700000 rows=3691809 buffered=8191 part=openwebtext-t5 +step=230750 lr=6.000000e-04 loss=1.9237 ce=1.9237 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.509848496236373e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003344334661960602, 'last_eos_top1': 0} +step=230800 lr=6.000000e-04 loss=2.3885 ce=2.3885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.216562830260955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022154278121888638, 'last_eos_top1': 0} +step=230850 lr=6.000000e-04 loss=3.0840 ce=3.0840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.329069684492424e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002614123048260808, 'last_eos_top1': 0} +step=230900 lr=6.000000e-04 loss=2.6422 ce=2.6422 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3512271834770218e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002243421506136656, 'last_eos_top1': 0} +step=230950 lr=6.000000e-04 loss=2.9952 ce=2.9952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4667016330349725e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020876950584352016, 'last_eos_top1': 0} +step=231000 lr=6.000000e-04 loss=2.7706 ce=2.7706 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.810146937321406e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002035677433013916, 'last_eos_top1': 0} +step=231050 lr=6.000000e-04 loss=2.1027 ce=2.1027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.9154008340556175e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017614744137972593, 'last_eos_top1': 0} +step=231100 lr=6.000000e-04 loss=3.0709 ce=3.0709 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.317143743217457e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003018764778971672, 'last_eos_top1': 0} +step=231150 lr=6.000000e-04 loss=1.9078 ce=1.9078 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.095089702284895e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003481645602732897, 'last_eos_top1': 0} +step=231200 lr=6.000000e-04 loss=2.3871 ce=2.3871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.202741810237058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028615195769816637, 'last_eos_top1': 0} +step=231250 lr=6.000000e-04 loss=2.5033 ce=2.5033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2665293979807757e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002719378564506769, 'last_eos_top1': 0} +step=231300 lr=6.000000e-04 loss=2.3326 ce=2.3326 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3907484571973328e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003794318065047264, 'last_eos_top1': 0} +step=231350 lr=6.000000e-04 loss=2.2253 ce=2.2253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.817966095695738e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004992505069822073, 'last_eos_top1': 0} +step=231400 lr=6.000000e-04 loss=2.4921 ce=2.4921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.681735961436061e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004385662265121937, 'last_eos_top1': 0} +step=231450 lr=6.000000e-04 loss=2.2898 ce=2.2898 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.685074029315729e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033095460385084152, 'last_eos_top1': 0} +step=231500 lr=6.000000e-04 loss=2.3553 ce=2.3553 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2249925021023955e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002950446680188179, 'last_eos_top1': 0} +step=231550 lr=6.000000e-04 loss=2.3574 ce=2.3574 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1902733604074456e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003511920105665922, 'last_eos_top1': 0} +step=231600 lr=6.000000e-04 loss=2.5875 ce=2.5875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0855113941943273e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036796564236283302, 'last_eos_top1': 0} +step=231650 lr=6.000000e-04 loss=2.2655 ce=2.2655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.584938091691583e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004314598627388477, 'last_eos_top1': 0} +step=231700 lr=6.000000e-04 loss=2.2103 ce=2.2103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.49785937537672e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005322059616446495, 'last_eos_top1': 0} +step=231750 lr=6.000000e-04 loss=2.2945 ce=2.2945 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.416867962689139e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003459338564425707, 'last_eos_top1': 0} +step=231800 lr=6.000000e-04 loss=2.3515 ce=2.3515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.194554326706566e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003552163951098919, 'last_eos_top1': 0} +step=231850 lr=6.000000e-04 loss=2.5377 ce=2.5377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4405037038377486e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004213106818497181, 'last_eos_top1': 0} +step=231900 lr=6.000000e-04 loss=3.0711 ce=3.0711 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1483679777011275e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037862237077206373, 'last_eos_top1': 0} +step=231950 lr=6.000000e-04 loss=2.6294 ce=2.6294 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.521566996932961e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030548889189958572, 'last_eos_top1': 0} +step=232000 lr=6.000000e-04 loss=2.2748 ce=2.2748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0553536412771791e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002668109955266118, 'last_eos_top1': 0} +step=232050 lr=6.000000e-04 loss=2.5475 ce=2.5475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7034388292813674e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0031284999568015337, 'last_eos_top1': 0} +step=232100 lr=6.000000e-04 loss=2.4850 ce=2.4850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0236401976726484e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004184000194072723, 'last_eos_top1': 0} +step=232150 lr=6.000000e-04 loss=2.7746 ce=2.7746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9399365555727854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004054993391036987, 'last_eos_top1': 0} +step=232200 lr=6.000000e-04 loss=3.3098 ce=3.3098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5908207539469004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004702838137745857, 'last_eos_top1': 0} +step=232250 lr=6.000000e-04 loss=2.2742 ce=2.2742 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.559049582690932e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003903693053871393, 'last_eos_top1': 0} +step=232300 lr=6.000000e-04 loss=2.7724 ce=2.7724 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0885325536946766e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025442256592214108, 'last_eos_top1': 0} +step=232350 lr=6.000000e-04 loss=3.0738 ce=3.0738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.96178165829042e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016619812231510878, 'last_eos_top1': 0} +step=232400 lr=6.000000e-04 loss=2.2808 ce=2.2808 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.349210474989377e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020558880642056465, 'last_eos_top1': 0} +step=232450 lr=6.000000e-04 loss=2.5924 ce=2.5924 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.590659828973003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0017909782472997904, 'last_eos_top1': 0} +step=232500 lr=6.000000e-04 loss=2.3661 ce=2.3661 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.98288056103047e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021686693653464317, 'last_eos_top1': 0} +step=232550 lr=6.000000e-04 loss=2.3763 ce=2.3763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2265714758541435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003636451903730631, 'last_eos_top1': 0} +step=232600 lr=6.000000e-04 loss=1.9925 ce=1.9925 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.720935637829825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003343902062624693, 'last_eos_top1': 0} +step=232650 lr=6.000000e-04 loss=2.3864 ce=2.3864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3259814295452088e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002056407742202282, 'last_eos_top1': 0} +step=232700 lr=6.000000e-04 loss=2.4727 ce=2.4727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.656246685248334e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021763681434094906, 'last_eos_top1': 0} +step=232750 lr=6.000000e-04 loss=2.6333 ce=2.6333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.170671202999074e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019971514120697975, 'last_eos_top1': 0} +step=232800 lr=6.000000e-04 loss=2.0758 ce=2.0758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.610592516954057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001789391739293933, 'last_eos_top1': 0} +step=232850 lr=6.000000e-04 loss=2.1100 ce=2.1100 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.486869435524568e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020660124719142914, 'last_eos_top1': 0} +step=232900 lr=6.000000e-04 loss=2.6173 ce=2.6173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0534331522649154e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002433719113469124, 'last_eos_top1': 0} +step=232950 lr=6.000000e-04 loss=1.7782 ce=1.7782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.707398824801203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016802579630166292, 'last_eos_top1': 0} +step=233000 lr=6.000000e-04 loss=2.4541 ce=2.4541 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.43218060961226e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002073123585432768, 'last_eos_top1': 0} +step=233050 lr=6.000000e-04 loss=2.1778 ce=2.1778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.074755558278412e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0019208414014428854, 'last_eos_top1': 0} +step=233100 lr=6.000000e-04 loss=2.5453 ce=2.5453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.2942018454778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002154961694031954, 'last_eos_top1': 0} +step=233150 lr=6.000000e-04 loss=2.4983 ce=2.4983 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5073575670830905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020476938225328922, 'last_eos_top1': 0} +step=233200 lr=6.000000e-04 loss=1.8944 ce=1.8944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.730763142404612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013578784419223666, 'last_eos_top1': 0} +step=233250 lr=6.000000e-04 loss=3.1707 ce=3.1707 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.213737030222546e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.000981127843260765, 'last_eos_top1': 0} +step=233300 lr=6.000000e-04 loss=2.8903 ce=2.8903 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.70243685413152e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011590970680117607, 'last_eos_top1': 0} +step=233350 lr=6.000000e-04 loss=2.3582 ce=2.3582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.142148903454654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0011292336275801063, 'last_eos_top1': 0} +step=233400 lr=6.000000e-04 loss=2.8466 ce=2.8466 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3743090676143765e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001681540859863162, 'last_eos_top1': 0} +step=233450 lr=6.000000e-04 loss=2.3013 ce=2.3013 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0262738214805722e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014580081915482879, 'last_eos_top1': 0} +step=233500 lr=6.000000e-04 loss=1.9286 ce=1.9286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.231643732869998e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002125325845554471, 'last_eos_top1': 0} +step=233550 lr=6.000000e-04 loss=2.7199 ce=2.7199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5563327653799206e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027616405859589577, 'last_eos_top1': 0} +step=233600 lr=6.000000e-04 loss=2.4105 ce=2.4105 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.683646567282267e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022257729433476925, 'last_eos_top1': 0} +step=233650 lr=6.000000e-04 loss=2.5856 ce=2.5856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.542560418485664e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0013785995543003082, 'last_eos_top1': 0} +step=233700 lr=6.000000e-04 loss=2.4253 ce=2.4253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2115360732423142e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004193751374259591, 'last_eos_top1': 0} +step=233750 lr=6.000000e-04 loss=2.3985 ce=2.3985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.826320607913658e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.000739602604880929, 'last_eos_top1': 0} +step=233800 lr=6.000000e-04 loss=2.8885 ce=2.8885 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.920267045032233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001139739528298378, 'last_eos_top1': 0} +step=233850 lr=6.000000e-04 loss=2.5565 ce=2.5565 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.661938409204595e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00047162926057353616, 'last_eos_top1': 0} +step=233900 lr=6.000000e-04 loss=2.2810 ce=2.2810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.147307198669296e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003843906451947987, 'last_eos_top1': 0} +step=233950 lr=6.000000e-04 loss=2.7938 ce=2.7938 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.094301296689082e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004978195065632463, 'last_eos_top1': 0} +step=234000 lr=6.000000e-04 loss=2.3625 ce=2.3625 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.453850462799892e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005302286590449512, 'last_eos_top1': 0} +step=234050 lr=6.000000e-04 loss=2.6844 ce=2.6844 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3933672461425886e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004972648457624018, 'last_eos_top1': 0} +step=234100 lr=6.000000e-04 loss=2.7944 ce=2.7944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7572700016899034e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00062131742015481, 'last_eos_top1': 0} +step=234150 lr=6.000000e-04 loss=3.2671 ce=3.2671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1674477718770504e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003380867128726095, 'last_eos_top1': 0} +step=234200 lr=6.000000e-04 loss=2.5287 ce=2.5287 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4567122889275197e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0007181341643445194, 'last_eos_top1': 0} +step=234250 lr=6.000000e-04 loss=2.2803 ce=2.2803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.612167685991153e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005796225741505623, 'last_eos_top1': 0} +step=234300 lr=6.000000e-04 loss=2.2725 ce=2.2725 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4659874371718615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0021232194267213345, 'last_eos_top1': 0} +step=234350 lr=6.000000e-04 loss=2.9185 ce=2.9185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.324842954403721e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008519720286130905, 'last_eos_top1': 0} +step=234400 lr=6.000000e-04 loss=2.8671 ce=2.8671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5114272173377685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0009745031129568815, 'last_eos_top1': 0} +step=234450 lr=6.000000e-04 loss=2.5941 ce=2.5941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8809405446518213e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0008430936140939593, 'last_eos_top1': 0} +step=234500 lr=6.000000e-04 loss=2.4183 ce=2.4183 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0513196684769355e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00035228620981797576, 'last_eos_top1': 0} +step=234550 lr=6.000000e-04 loss=2.9914 ce=2.9914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3536985534301493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00036674150032922626, 'last_eos_top1': 0} +step=234600 lr=6.000000e-04 loss=1.6588 ce=1.6588 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.409882866544649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00024899147683754563, 'last_eos_top1': 0} +step=234650 lr=6.000000e-04 loss=2.7705 ce=2.7705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1952719432883896e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006239420035853982, 'last_eos_top1': 0} +step=234700 lr=6.000000e-04 loss=2.2320 ce=2.2320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.682889529154636e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0005937216337770224, 'last_eos_top1': 0} +step=234750 lr=6.000000e-04 loss=2.1376 ce=2.1376 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0286454198649153e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004315899277571589, 'last_eos_top1': 0} +step=234800 lr=6.000000e-04 loss=2.8269 ce=2.8269 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.76398223428987e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0003452258533798158, 'last_eos_top1': 0} +step=234850 lr=6.000000e-04 loss=1.7221 ce=1.7221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.652637810271699e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00038104417035356164, 'last_eos_top1': 0} +step=234900 lr=6.000000e-04 loss=2.1583 ce=2.1583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7853810277301818e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006514330161735415, 'last_eos_top1': 0} +step=234950 lr=6.000000e-04 loss=2.5948 ce=2.5948 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3220132132119033e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0004444005899131298, 'last_eos_top1': 0} +step=235000 lr=6.000000e-04 loss=2.4264 ce=2.4264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.54189409740502e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00027297958149574697, 'last_eos_top1': 0} +step=235050 lr=6.000000e-04 loss=2.3479 ce=2.3479 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.6196529435983393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0002981256111524999, 'last_eos_top1': 0} +step=235100 lr=6.000000e-04 loss=2.7456 ce=2.7456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.527745204541134e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00044550347956828773, 'last_eos_top1': 0} +step=235150 lr=6.000000e-04 loss=2.0507 ce=2.0507 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8351861298433505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0006087956135161221, 'last_eos_top1': 0} +step=235200 lr=6.000000e-04 loss=2.3993 ce=2.3993 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.809793143882416e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001265808823518455, 'last_eos_top1': 0} +step=235250 lr=6.000000e-04 loss=2.6814 ce=2.6814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.364651769923512e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0027211587876081467, 'last_eos_top1': 0} +step=235300 lr=6.000000e-04 loss=2.6286 ce=2.6286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.698450306430459e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.001979970373213291, 'last_eos_top1': 0} +step=235350 lr=6.000000e-04 loss=2.9577 ce=2.9577 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2268030332052149e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003778850194066763, 'last_eos_top1': 0} +step=235400 lr=6.000000e-04 loss=2.6096 ce=2.6096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.4181135308463126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033059187699109316, 'last_eos_top1': 0} +step=235450 lr=6.000000e-04 loss=2.5017 ce=2.5017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.774981319613289e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002043950604274869, 'last_eos_top1': 0} +step=235500 lr=6.000000e-04 loss=2.5397 ce=2.5397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.399127520737238e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020107850432395935, 'last_eos_top1': 0} +step=235550 lr=6.000000e-04 loss=1.8265 ce=1.8265 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1460600944701582e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0029333382844924927, 'last_eos_top1': 0} +step=235600 lr=6.000000e-04 loss=2.4936 ce=2.4936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.19695463986136e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016282612923532724, 'last_eos_top1': 0} +step=235650 lr=6.000000e-04 loss=2.5299 ce=2.5299 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.296525624638889e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0016953549347817898, 'last_eos_top1': 0} +step=235700 lr=6.000000e-04 loss=2.9114 ce=2.9114 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.005296538409311e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00109010748565197, 'last_eos_top1': 0} +step=235750 lr=6.000000e-04 loss=2.8610 ce=2.8610 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.89822206570534e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0015693773748353124, 'last_eos_top1': 0} +step=235800 lr=6.000000e-04 loss=2.1463 ce=2.1463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0441316590004135e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0014220753218978643, 'last_eos_top1': 0} +step=235850 lr=6.000000e-04 loss=2.0022 ce=2.0022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2355814760667272e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.001785696018487215, 'last_eos_top1': 0} +step=235900 lr=6.000000e-04 loss=2.5352 ce=2.5352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.716534244333161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018337084911763668, 'last_eos_top1': 0} +step=235950 lr=6.000000e-04 loss=2.2192 ce=2.2192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2643933587241918e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002509597921743989, 'last_eos_top1': 0} +step=236000 lr=6.000000e-04 loss=2.4467 ce=2.4467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.614293796592392e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0025195926427841187, 'last_eos_top1': 0} +step=236050 lr=6.000000e-04 loss=2.5636 ce=2.5636 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3709995982935652e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0020834298338741064, 'last_eos_top1': 0} +step=236100 lr=6.000000e-04 loss=2.4939 ce=2.4939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.071111091732746e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018845981685444713, 'last_eos_top1': 0} +step=236150 lr=6.000000e-04 loss=2.2448 ce=2.2448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.74888246471528e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0022088841069489717, 'last_eos_top1': 0} +step=236200 lr=6.000000e-04 loss=2.5173 ce=2.5173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.782835847232491e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003643664764240384, 'last_eos_top1': 0} +step=236250 lr=6.000000e-04 loss=1.9179 ce=1.9179 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7305804906063713e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003321679774671793, 'last_eos_top1': 0} +step=236300 lr=6.000000e-04 loss=2.5446 ce=2.5446 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.428694486094173e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028413834515959024, 'last_eos_top1': 0} +step=236350 lr=6.000000e-04 loss=2.1727 ce=2.1727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9455233743647113e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003347235731780529, 'last_eos_top1': 0} +step=236400 lr=6.000000e-04 loss=2.9907 ce=2.9907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8295610061613843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0035473438911139965, 'last_eos_top1': 0} +step=236450 lr=6.000000e-04 loss=1.9258 ce=1.9258 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3603664885740727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004170170519500971, 'last_eos_top1': 0} +step=236500 lr=6.000000e-04 loss=1.9848 ce=1.9848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.059342594089685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002861957997083664, 'last_eos_top1': 0} +step=236550 lr=6.000000e-04 loss=2.9077 ce=2.9077 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.11990809981944e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028126367833465338, 'last_eos_top1': 0} +step=236600 lr=6.000000e-04 loss=2.5740 ce=2.5740 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.215383392467629e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033785649575293064, 'last_eos_top1': 0} +step=236650 lr=6.000000e-04 loss=2.5966 ce=2.5966 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.160426745627774e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006137858144938946, 'last_eos_top1': 0} +step=236700 lr=6.000000e-04 loss=2.6593 ce=2.6593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3146279343345668e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005496269091963768, 'last_eos_top1': 0} +step=236750 lr=6.000000e-04 loss=2.6505 ce=2.6505 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.821418305160478e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005069089122116566, 'last_eos_top1': 0} +step=236800 lr=6.000000e-04 loss=2.3519 ce=2.3519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7359758203383535e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005178976804018021, 'last_eos_top1': 0} +step=236850 lr=6.000000e-04 loss=2.5873 ce=2.5873 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.495166755688842e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0043859900906682014, 'last_eos_top1': 0} +step=236900 lr=6.000000e-04 loss=2.4156 ce=2.4156 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.040239092428237e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038860496133565903, 'last_eos_top1': 0} +step=236950 lr=6.000000e-04 loss=2.5776 ce=2.5776 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.312625712074805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002460191957652569, 'last_eos_top1': 0} +[elf_hfds direct rank=6 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=4 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=3800000 rows=3791809 buffered=8191 part=openwebtext-t5 +step=237000 lr=6.000000e-04 loss=2.2431 ce=2.2431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.797663784003817e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0028163171373307705, 'last_eos_top1': 0} +step=237050 lr=6.000000e-04 loss=2.8911 ce=2.8911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.073064595402684e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038244626484811306, 'last_eos_top1': 0} +step=237100 lr=6.000000e-04 loss=2.9409 ce=2.9409 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.519935904070735e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0042416248470544815, 'last_eos_top1': 0} +step=237150 lr=6.000000e-04 loss=2.2572 ce=2.2572 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.893818408570951e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032095550559461117, 'last_eos_top1': 0} +step=237200 lr=6.000000e-04 loss=2.9972 ce=2.9972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.777698904945282e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003241822123527527, 'last_eos_top1': 0} +step=237250 lr=6.000000e-04 loss=2.5471 ce=2.5471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.31006036605686e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044840360060334206, 'last_eos_top1': 0} +step=237300 lr=6.000000e-04 loss=2.4743 ce=2.4743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1324767658370547e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006382724270224571, 'last_eos_top1': 0} +step=237350 lr=6.000000e-04 loss=2.5969 ce=2.5969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2540571333374828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005407417193055153, 'last_eos_top1': 0} +step=237400 lr=6.000000e-04 loss=2.3600 ce=2.3600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0616091205738485e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00626948568969965, 'last_eos_top1': 0} +step=237450 lr=6.000000e-04 loss=2.6181 ce=2.6181 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.524983397568576e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007740280125290155, 'last_eos_top1': 0} +step=237500 lr=6.000000e-04 loss=2.4531 ce=2.4531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.070701728167478e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00687768030911684, 'last_eos_top1': 0} +step=237550 lr=6.000000e-04 loss=2.6931 ce=2.6931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.343578151136171e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007501751184463501, 'last_eos_top1': 0} +step=237600 lr=6.000000e-04 loss=2.1534 ce=2.1534 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1983949661953375e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006026692688465118, 'last_eos_top1': 0} +step=237650 lr=6.000000e-04 loss=2.3809 ce=2.3809 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0328452844987623e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0057068136520683765, 'last_eos_top1': 0} +step=237700 lr=6.000000e-04 loss=2.7585 ce=2.7585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.514677236031275e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007133046165108681, 'last_eos_top1': 0} +step=237750 lr=6.000000e-04 loss=2.5421 ce=2.5421 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6636498912703246e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007802009116858244, 'last_eos_top1': 0} +step=237800 lr=6.000000e-04 loss=2.4949 ce=2.4949 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.00555041374173e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008478580042719841, 'last_eos_top1': 0} +step=237850 lr=6.000000e-04 loss=2.3133 ce=2.3133 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.244437018816825e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010615658946335316, 'last_eos_top1': 0} +step=237900 lr=6.000000e-04 loss=2.4991 ce=2.4991 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.5220960601000115e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007261198479682207, 'last_eos_top1': 0} +step=237950 lr=6.000000e-04 loss=2.7730 ce=2.7730 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5217921827570535e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006571193225681782, 'last_eos_top1': 0} +step=238000 lr=6.000000e-04 loss=2.5873 ce=2.5873 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.243103704648092e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00586132425814867, 'last_eos_top1': 0} +step=238050 lr=6.000000e-04 loss=2.2514 ce=2.2514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.280061769648455e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010388582944869995, 'last_eos_top1': 0} +step=238100 lr=6.000000e-04 loss=2.0494 ce=2.0494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.453776899841614e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010178131982684135, 'last_eos_top1': 0} +step=238150 lr=6.000000e-04 loss=2.3504 ce=2.3504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2781505574821495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010059883818030357, 'last_eos_top1': 0} +step=238200 lr=6.000000e-04 loss=2.7025 ce=2.7025 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.174568701069802e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011976880952715874, 'last_eos_top1': 0} +step=238250 lr=6.000000e-04 loss=2.4003 ce=2.4003 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.086667857132852e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016983192414045334, 'last_eos_top1': 0} +step=238300 lr=6.000000e-04 loss=2.6155 ce=2.6155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.675858766247984e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015804113820195198, 'last_eos_top1': 0} +step=238350 lr=6.000000e-04 loss=2.0839 ce=2.0839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9520663045113906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014363185502588749, 'last_eos_top1': 0} +step=238400 lr=6.000000e-04 loss=2.3698 ce=2.3698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.847116648918018e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015332928858697414, 'last_eos_top1': 0} +step=238450 lr=6.000000e-04 loss=2.4787 ce=2.4787 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5592464478686452e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012434188276529312, 'last_eos_top1': 0} +step=238500 lr=6.000000e-04 loss=2.6637 ce=2.6637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1313674804114271e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011372365057468414, 'last_eos_top1': 0} +step=238550 lr=6.000000e-04 loss=2.2014 ce=2.2014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2146573681093287e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010241847485303879, 'last_eos_top1': 0} +step=238600 lr=6.000000e-04 loss=2.5475 ce=2.5475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1170868674525991e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01099635474383831, 'last_eos_top1': 0} +step=238650 lr=6.000000e-04 loss=2.3524 ce=2.3524 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1311774869682267e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010227525606751442, 'last_eos_top1': 0} +step=238700 lr=6.000000e-04 loss=2.0155 ce=2.0155 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4489207135047764e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008731706067919731, 'last_eos_top1': 0} +step=238750 lr=6.000000e-04 loss=2.8071 ce=2.8071 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.933677776833065e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008404524996876717, 'last_eos_top1': 0} +step=238800 lr=6.000000e-04 loss=3.1482 ce=3.1482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.730530680622905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0077137211337685585, 'last_eos_top1': 0} +step=238850 lr=6.000000e-04 loss=2.6595 ce=2.6595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.660082853049971e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00839601643383503, 'last_eos_top1': 0} +step=238900 lr=6.000000e-04 loss=2.2377 ce=2.2377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4658191503258422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00644448259845376, 'last_eos_top1': 0} +step=238950 lr=6.000000e-04 loss=2.1411 ce=2.1411 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4637226740887854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006655228324234486, 'last_eos_top1': 0} +step=239000 lr=6.000000e-04 loss=2.0850 ce=2.0850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2930600860272534e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00816253386437893, 'last_eos_top1': 0} +step=239050 lr=6.000000e-04 loss=2.5924 ce=2.5924 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4569588529411703e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006098026409745216, 'last_eos_top1': 0} +step=239100 lr=6.000000e-04 loss=2.4181 ce=2.4181 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3449487596517429e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004145465791225433, 'last_eos_top1': 0} +step=239150 lr=6.000000e-04 loss=2.6333 ce=2.6333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.805446299840696e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044761402532458305, 'last_eos_top1': 0} +step=239200 lr=6.000000e-04 loss=2.7818 ce=2.7818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.936801921459846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005338548682630062, 'last_eos_top1': 0} +step=239250 lr=6.000000e-04 loss=2.8096 ce=2.8096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9641593098640442e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006743405479937792, 'last_eos_top1': 0} +step=239300 lr=6.000000e-04 loss=2.7466 ce=2.7466 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4414097677217796e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007276561111211777, 'last_eos_top1': 0} +step=239350 lr=6.000000e-04 loss=2.6392 ce=2.6392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3284762644616421e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006730520632117987, 'last_eos_top1': 0} +step=239400 lr=6.000000e-04 loss=2.6911 ce=2.6911 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.918380783346947e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00584575068205595, 'last_eos_top1': 0} +step=239450 lr=6.000000e-04 loss=2.7059 ce=2.7059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3168850273359567e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005856768228113651, 'last_eos_top1': 0} +step=239500 lr=6.000000e-04 loss=2.5513 ce=2.5513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3512855730368756e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006301491521298885, 'last_eos_top1': 0} +step=239550 lr=6.000000e-04 loss=1.9918 ce=1.9918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5377132513094693e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006379328202456236, 'last_eos_top1': 0} +step=239600 lr=6.000000e-04 loss=2.7213 ce=2.7213 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.227417649526615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00694294273853302, 'last_eos_top1': 0} +step=239650 lr=6.000000e-04 loss=2.3932 ce=2.3932 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6458845493616536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00552050257101655, 'last_eos_top1': 0} +step=239700 lr=6.000000e-04 loss=2.3186 ce=2.3186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5662913938285783e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006696890573948622, 'last_eos_top1': 0} +step=239750 lr=6.000000e-04 loss=2.6281 ce=2.6281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.619486263545696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005071957595646381, 'last_eos_top1': 0} +step=239800 lr=6.000000e-04 loss=2.5079 ce=2.5079 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.670734491199255e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004286201670765877, 'last_eos_top1': 0} +step=239850 lr=6.000000e-04 loss=2.6489 ce=2.6489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4932313206372783e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005871205125004053, 'last_eos_top1': 0} +step=239900 lr=6.000000e-04 loss=2.5332 ce=2.5332 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2871087164967321e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006920638494193554, 'last_eos_top1': 0} +step=239950 lr=6.000000e-04 loss=2.4019 ce=2.4019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.115351369662676e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007428908720612526, 'last_eos_top1': 0} +step=240000 lr=6.000000e-04 loss=2.7197 ce=2.7197 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3278828191687353e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0059934016317129135, 'last_eos_top1': 0} +step=240050 lr=6.000000e-04 loss=1.7513 ce=1.7513 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5179766705841757e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004658405669033527, 'last_eos_top1': 0} +step=240100 lr=6.000000e-04 loss=2.9340 ce=2.9340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.236222477833508e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005017257295548916, 'last_eos_top1': 0} +step=240150 lr=6.000000e-04 loss=2.1161 ce=2.1161 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.926628546149004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007570646703243256, 'last_eos_top1': 0} +step=240200 lr=6.000000e-04 loss=2.5124 ce=2.5124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1343526239215862e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007053449749946594, 'last_eos_top1': 0} +step=240250 lr=6.000000e-04 loss=2.9255 ce=2.9255 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.737631909141783e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065829018130898476, 'last_eos_top1': 0} +step=240300 lr=6.000000e-04 loss=3.0010 ce=3.0010 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.352385985839646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005130154546350241, 'last_eos_top1': 0} +step=240350 lr=6.000000e-04 loss=2.3296 ce=2.3296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.268463529413566e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005349271930754185, 'last_eos_top1': 0} +step=240400 lr=6.000000e-04 loss=2.6224 ce=2.6224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7662376194493845e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005947968922555447, 'last_eos_top1': 0} +step=240450 lr=6.000000e-04 loss=2.2094 ce=2.2094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8401819033897482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008689496666193008, 'last_eos_top1': 0} +step=240500 lr=6.000000e-04 loss=2.6167 ce=2.6167 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3720893548452295e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007504483684897423, 'last_eos_top1': 0} +step=240550 lr=6.000000e-04 loss=2.3632 ce=2.3632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.189163933508098e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00876937247812748, 'last_eos_top1': 0} +step=240600 lr=6.000000e-04 loss=2.2904 ce=2.2904 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.58558063453529e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007201087661087513, 'last_eos_top1': 0} +step=240650 lr=6.000000e-04 loss=2.6803 ce=2.6803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8863107470679097e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006050603464245796, 'last_eos_top1': 0} +step=240700 lr=6.000000e-04 loss=2.8646 ce=2.8646 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.902276360662654e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008404415100812912, 'last_eos_top1': 0} +step=240750 lr=6.000000e-04 loss=2.7726 ce=2.7726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.496956408838741e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009063716977834702, 'last_eos_top1': 0} +step=240800 lr=6.000000e-04 loss=2.3743 ce=2.3743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5095056369318627e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0059667606838047504, 'last_eos_top1': 0} +step=240850 lr=6.000000e-04 loss=2.4183 ce=2.4183 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.86150451656431e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005648735910654068, 'last_eos_top1': 0} +step=240900 lr=6.000000e-04 loss=2.2209 ce=2.2209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.536435987276491e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006030671298503876, 'last_eos_top1': 0} +step=240950 lr=6.000000e-04 loss=1.9200 ce=1.9200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8545266104629263e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00591303501278162, 'last_eos_top1': 0} +step=241000 lr=6.000000e-04 loss=3.0393 ce=3.0393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.880400618072599e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0049048978835344315, 'last_eos_top1': 0} +step=241050 lr=6.000000e-04 loss=2.4136 ce=2.4136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4441753592109308e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003993654623627663, 'last_eos_top1': 0} +step=241100 lr=6.000000e-04 loss=2.5928 ce=2.5928 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5050770748057403e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007359417155385017, 'last_eos_top1': 0} +step=241150 lr=6.000000e-04 loss=2.7476 ce=2.7476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.511713869869709e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010172422975301743, 'last_eos_top1': 0} +step=241200 lr=6.000000e-04 loss=2.4366 ce=2.4366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.961807230254635e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00987496878951788, 'last_eos_top1': 0} +step=241250 lr=6.000000e-04 loss=2.4552 ce=2.4552 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0643490643124096e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0073217772878706455, 'last_eos_top1': 0} +step=241300 lr=6.000000e-04 loss=2.2737 ce=2.2737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8733344404608943e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008626949042081833, 'last_eos_top1': 0} +step=241350 lr=6.000000e-04 loss=2.8369 ce=2.8369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.045443221111782e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008276930078864098, 'last_eos_top1': 0} +step=241400 lr=6.000000e-04 loss=2.1592 ce=2.1592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.99793225369649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008599840104579926, 'last_eos_top1': 0} +step=241450 lr=6.000000e-04 loss=2.1404 ce=2.1404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.337700248579495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007961923256516457, 'last_eos_top1': 0} +step=241500 lr=6.000000e-04 loss=2.0271 ce=2.0271 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6534366295672953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008546387776732445, 'last_eos_top1': 0} +step=241550 lr=6.000000e-04 loss=2.1705 ce=2.1705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.187755671911873e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006132690701633692, 'last_eos_top1': 0} +step=241600 lr=6.000000e-04 loss=2.7754 ce=2.7754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2173262550495565e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006507122423499823, 'last_eos_top1': 0} +step=241650 lr=6.000000e-04 loss=2.0706 ce=2.0706 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.120266046607867e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00713239423930645, 'last_eos_top1': 0} +step=241700 lr=6.000000e-04 loss=2.4320 ce=2.4320 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3977662092656828e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0073565891943871975, 'last_eos_top1': 0} +step=241750 lr=6.000000e-04 loss=2.8630 ce=2.8630 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1632611606037244e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011092361062765121, 'last_eos_top1': 0} +step=241800 lr=6.000000e-04 loss=2.8593 ce=2.8593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.69979794009123e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013430215418338776, 'last_eos_top1': 0} +step=241850 lr=6.000000e-04 loss=2.4793 ce=2.4793 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5053652279893868e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011292826384305954, 'last_eos_top1': 0} +step=241900 lr=6.000000e-04 loss=2.0954 ce=2.0954 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.9172155084088445e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010203123092651367, 'last_eos_top1': 0} +step=241950 lr=6.000000e-04 loss=3.2336 ce=3.2336 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0093830244150013e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00688434112817049, 'last_eos_top1': 0} +step=242000 lr=6.000000e-04 loss=2.7150 ce=2.7150 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5481988157262094e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006320773158222437, 'last_eos_top1': 0} +step=242050 lr=6.000000e-04 loss=2.9346 ce=2.9346 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.325886453036219e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008055703714489937, 'last_eos_top1': 0} +step=242100 lr=6.000000e-04 loss=2.5749 ce=2.5749 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0911311619565822e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008663524873554707, 'last_eos_top1': 0} +step=242150 lr=6.000000e-04 loss=2.7414 ce=2.7414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.24001415114617e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00747354980558157, 'last_eos_top1': 0} +step=242200 lr=6.000000e-04 loss=3.2499 ce=3.2499 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8446978831198066e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00632073637098074, 'last_eos_top1': 0} +step=242250 lr=6.000000e-04 loss=2.2898 ce=2.2898 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7718973797163926e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00670916773378849, 'last_eos_top1': 0} +step=242300 lr=6.000000e-04 loss=2.1736 ce=2.1736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0728424715343863e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005566245876252651, 'last_eos_top1': 0} +step=242350 lr=6.000000e-04 loss=2.1200 ce=2.1200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1847268979181536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006480314768850803, 'last_eos_top1': 0} +step=242400 lr=6.000000e-04 loss=2.9187 ce=2.9187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.000095264520496e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008171175606548786, 'last_eos_top1': 0} +step=242450 lr=6.000000e-04 loss=2.7746 ce=2.7746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2857288058730774e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00836951658129692, 'last_eos_top1': 0} +step=242500 lr=6.000000e-04 loss=2.5679 ce=2.5679 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8169237591791898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009551947936415672, 'last_eos_top1': 0} +step=242550 lr=6.000000e-04 loss=2.7094 ce=2.7094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.237782973679714e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007245603017508984, 'last_eos_top1': 0} +step=242600 lr=6.000000e-04 loss=2.7348 ce=2.7348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.945307748334017e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0062177060171961784, 'last_eos_top1': 0} +step=242650 lr=6.000000e-04 loss=2.7068 ce=2.7068 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1073898096801713e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00855378620326519, 'last_eos_top1': 0} +step=242700 lr=6.000000e-04 loss=2.7193 ce=2.7193 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3954069800092839e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0063813915476202965, 'last_eos_top1': 0} +step=242750 lr=6.000000e-04 loss=2.2474 ce=2.2474 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1858143807330634e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008216704241931438, 'last_eos_top1': 0} +step=242800 lr=6.000000e-04 loss=2.3089 ce=2.3089 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.905307337641716e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00778742041438818, 'last_eos_top1': 0} +step=242850 lr=6.000000e-04 loss=2.2514 ce=2.2514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9317676560604014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010142543353140354, 'last_eos_top1': 0} +step=242900 lr=6.000000e-04 loss=2.8833 ce=2.8833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1068646017520223e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008015584200620651, 'last_eos_top1': 0} +step=242950 lr=6.000000e-04 loss=3.0702 ce=3.0702 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6037931345636025e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010218393057584763, 'last_eos_top1': 0} +step=243000 lr=6.000000e-04 loss=2.8015 ce=2.8015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2215686183481012e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008191500790417194, 'last_eos_top1': 0} +step=243050 lr=6.000000e-04 loss=2.2613 ce=2.2613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.963375431543682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007778058759868145, 'last_eos_top1': 0} +step=243100 lr=6.000000e-04 loss=2.5227 ce=2.5227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0077917977469042e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008527828380465508, 'last_eos_top1': 0} +step=243150 lr=6.000000e-04 loss=2.5697 ce=2.5697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.683662599651143e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0077437786385416985, 'last_eos_top1': 0} +step=243200 lr=6.000000e-04 loss=2.2994 ce=2.2994 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.110598522151122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007787424139678478, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=3900000 rows=3891809 buffered=8191 part=openwebtext-t5 +step=243250 lr=6.000000e-04 loss=2.8076 ce=2.8076 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.815415640128776e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0069758775644004345, 'last_eos_top1': 0} +step=243300 lr=6.000000e-04 loss=2.6547 ce=2.6547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7070127796614543e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008914347738027573, 'last_eos_top1': 0} +step=243350 lr=6.000000e-04 loss=2.2732 ce=2.2732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0487981853657402e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00849117524921894, 'last_eos_top1': 0} +step=243400 lr=6.000000e-04 loss=1.7985 ce=1.7985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4805169485043734e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006381081882864237, 'last_eos_top1': 0} +step=243450 lr=6.000000e-04 loss=2.3789 ce=2.3789 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4800050848862156e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0075611695647239685, 'last_eos_top1': 0} +step=243500 lr=6.000000e-04 loss=2.9671 ce=2.9671 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.514366527379025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007785400375723839, 'last_eos_top1': 0} +step=243550 lr=6.000000e-04 loss=2.2685 ce=2.2685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.713158993283287e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008908553048968315, 'last_eos_top1': 0} +step=243600 lr=6.000000e-04 loss=1.6179 ce=1.6179 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0213401765213348e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011121783405542374, 'last_eos_top1': 0} +step=243650 lr=6.000000e-04 loss=2.3194 ce=2.3194 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.6279586715390906e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010770134627819061, 'last_eos_top1': 0} +step=243700 lr=6.000000e-04 loss=2.5841 ce=2.5841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.149238505604444e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008977288380265236, 'last_eos_top1': 0} +step=243750 lr=6.000000e-04 loss=2.2613 ce=2.2613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4931729310774244e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010736357420682907, 'last_eos_top1': 0} +step=243800 lr=6.000000e-04 loss=2.8435 ce=2.8435 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.405463631788734e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010242952033877373, 'last_eos_top1': 0} +step=243850 lr=6.000000e-04 loss=2.2322 ce=2.2322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2764345228788443e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009051285684108734, 'last_eos_top1': 0} +step=243900 lr=6.000000e-04 loss=2.3650 ce=2.3650 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4752475181012414e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008964985609054565, 'last_eos_top1': 0} +step=243950 lr=6.000000e-04 loss=2.0711 ce=2.0711 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8653103072429076e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008273730054497719, 'last_eos_top1': 0} +step=244000 lr=6.000000e-04 loss=2.4389 ce=2.4389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5114104826352559e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0060052527114748955, 'last_eos_top1': 0} +step=244050 lr=6.000000e-04 loss=2.7144 ce=2.7144 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0085481335408986e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006502694450318813, 'last_eos_top1': 0} +step=244100 lr=6.000000e-04 loss=2.9221 ce=2.9221 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4094212019699626e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007123597897589207, 'last_eos_top1': 0} +step=244150 lr=6.000000e-04 loss=2.6348 ce=2.6348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.787199254176812e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006771832704544067, 'last_eos_top1': 0} +step=244200 lr=6.000000e-04 loss=2.6762 ce=2.6762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.25900605780771e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011086856946349144, 'last_eos_top1': 0} +step=244250 lr=6.000000e-04 loss=2.5902 ce=2.5902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3040177691436838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014750126749277115, 'last_eos_top1': 0} +step=244300 lr=6.000000e-04 loss=2.7366 ce=2.7366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.485688572865911e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013687316328287125, 'last_eos_top1': 0} +step=244350 lr=6.000000e-04 loss=2.0765 ce=2.0765 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4915138308424503e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011609066277742386, 'last_eos_top1': 0} +step=244400 lr=6.000000e-04 loss=2.7236 ce=2.7236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1833118807990104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011596312746405602, 'last_eos_top1': 0} +step=244450 lr=6.000000e-04 loss=2.8153 ce=2.8153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9349816284375265e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008800562471151352, 'last_eos_top1': 0} +step=244500 lr=6.000000e-04 loss=2.5930 ce=2.5930 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8653117876965553e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006553420796990395, 'last_eos_top1': 0} +step=244550 lr=6.000000e-04 loss=2.7543 ce=2.7543 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.843613790697418e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065930685959756374, 'last_eos_top1': 0} +step=244600 lr=6.000000e-04 loss=2.9051 ce=2.9051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9254239305155352e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008092382922768593, 'last_eos_top1': 0} +step=244650 lr=6.000000e-04 loss=2.5385 ce=2.5385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7808786398964003e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008043983019888401, 'last_eos_top1': 0} +step=244700 lr=6.000000e-04 loss=2.7521 ce=2.7521 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.974682714324445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009334725327789783, 'last_eos_top1': 0} +step=244750 lr=6.000000e-04 loss=2.2456 ce=2.2456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4055495739739854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008711381815373898, 'last_eos_top1': 0} +step=244800 lr=6.000000e-04 loss=2.2756 ce=2.2756 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6107933333842084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009779558517038822, 'last_eos_top1': 0} +step=244850 lr=6.000000e-04 loss=3.1124 ce=3.1124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.21824079239741e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007697365712374449, 'last_eos_top1': 0} +step=244900 lr=6.000000e-04 loss=2.7116 ce=2.7116 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.479009319562465e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006107294000685215, 'last_eos_top1': 0} +step=244950 lr=6.000000e-04 loss=2.5485 ce=2.5485 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.139134448952973e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004930236842483282, 'last_eos_top1': 0} +step=245000 lr=6.000000e-04 loss=2.6259 ce=2.6259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6861704100156203e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00517586525529623, 'last_eos_top1': 0} +step=245050 lr=6.000000e-04 loss=2.6948 ce=2.6948 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5185059965006076e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004115947522222996, 'last_eos_top1': 0} +step=245100 lr=6.000000e-04 loss=2.5042 ce=2.5042 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8617761927307583e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004592024255543947, 'last_eos_top1': 0} +step=245150 lr=6.000000e-04 loss=2.4736 ce=2.4736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.634849109512288e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004285336472094059, 'last_eos_top1': 0} +step=245200 lr=6.000000e-04 loss=2.3366 ce=2.3366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.107250111293979e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004635302349925041, 'last_eos_top1': 0} +step=245250 lr=6.000000e-04 loss=2.1849 ce=2.1849 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0260615454171784e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003718311432749033, 'last_eos_top1': 0} +step=245300 lr=6.000000e-04 loss=2.8910 ce=2.8910 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.960326911415905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003911480773240328, 'last_eos_top1': 0} +step=245350 lr=6.000000e-04 loss=2.6732 ce=2.6732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6934041923377663e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003923861309885979, 'last_eos_top1': 0} +step=245400 lr=6.000000e-04 loss=2.2942 ce=2.2942 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.747541869524866e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0033073306549340487, 'last_eos_top1': 0} +step=245450 lr=6.000000e-04 loss=2.3072 ce=2.3072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.828569323464762e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004754846449941397, 'last_eos_top1': 0} +step=245500 lr=6.000000e-04 loss=2.8527 ce=2.8527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5135116882447619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00556968804448843, 'last_eos_top1': 0} +step=245550 lr=6.000000e-04 loss=2.9695 ce=2.9695 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6911544662434608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004567089024931192, 'last_eos_top1': 0} +step=245600 lr=6.000000e-04 loss=2.4909 ce=2.4909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.74449903878849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0048219491727650166, 'last_eos_top1': 0} +step=245650 lr=6.000000e-04 loss=2.4014 ce=2.4014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0820967620238662e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007789766415953636, 'last_eos_top1': 0} +step=245700 lr=6.000000e-04 loss=2.4220 ce=2.4220 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8163523893454112e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010741325095295906, 'last_eos_top1': 0} +step=245750 lr=6.000000e-04 loss=2.9018 ce=2.9018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4405272850126494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010302734561264515, 'last_eos_top1': 0} +step=245800 lr=6.000000e-04 loss=1.9305 ce=1.9305 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.312811248993967e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007394098211079836, 'last_eos_top1': 0} +step=245850 lr=6.000000e-04 loss=1.9705 ce=1.9705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.662668728386052e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005955865141004324, 'last_eos_top1': 0} +step=245900 lr=6.000000e-04 loss=2.4142 ce=2.4142 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.009328116197139e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004225492011755705, 'last_eos_top1': 0} +step=245950 lr=6.000000e-04 loss=1.8227 ce=1.8227 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3463013349100947e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0075944941490888596, 'last_eos_top1': 0} +step=246000 lr=6.000000e-04 loss=2.3981 ce=2.3981 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7186430341098458e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008632213808596134, 'last_eos_top1': 0} +step=246050 lr=6.000000e-04 loss=2.9318 ce=2.9318 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2888484535505995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006418738514184952, 'last_eos_top1': 0} +step=246100 lr=6.000000e-04 loss=2.2448 ce=2.2448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2222080840729177e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011866518296301365, 'last_eos_top1': 0} +step=246150 lr=6.000000e-04 loss=2.5587 ce=2.5587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.957616511092056e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010509221814572811, 'last_eos_top1': 0} +step=246200 lr=6.000000e-04 loss=2.4127 ce=2.4127 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.927879546012264e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007627568673342466, 'last_eos_top1': 0} +step=246250 lr=6.000000e-04 loss=2.5653 ce=2.5653 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.5356604207190685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007247529923915863, 'last_eos_top1': 0} +step=246300 lr=6.000000e-04 loss=2.2281 ce=2.2281 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.255341526819393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009036154486238956, 'last_eos_top1': 0} +step=246350 lr=6.000000e-04 loss=2.4918 ce=2.4918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.505234523501713e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007824869826436043, 'last_eos_top1': 0} +step=246400 lr=6.000000e-04 loss=2.5816 ce=2.5816 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.008995791897178e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008125454187393188, 'last_eos_top1': 0} +step=246450 lr=6.000000e-04 loss=2.3669 ce=2.3669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2204366612422746e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010250439867377281, 'last_eos_top1': 0} +step=246500 lr=6.000000e-04 loss=2.4875 ce=2.4875 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3870230759494007e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007820026949048042, 'last_eos_top1': 0} +step=246550 lr=6.000000e-04 loss=2.6997 ce=2.6997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.69422149460297e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008986387401819229, 'last_eos_top1': 0} +step=246600 lr=6.000000e-04 loss=2.5091 ce=2.5091 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.982264388469048e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008312052115797997, 'last_eos_top1': 0} +step=246650 lr=6.000000e-04 loss=2.2260 ce=2.2260 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6793961549410596e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011185968294739723, 'last_eos_top1': 0} +step=246700 lr=6.000000e-04 loss=2.7992 ce=2.7992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.434092155250255e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00956454686820507, 'last_eos_top1': 0} +step=246750 lr=6.000000e-04 loss=3.0364 ce=3.0364 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.130274414317682e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011150112375617027, 'last_eos_top1': 0} +step=246800 lr=6.000000e-04 loss=2.2956 ce=2.2956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.626510185014922e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010387731716036797, 'last_eos_top1': 0} +step=246850 lr=6.000000e-04 loss=2.2893 ce=2.2893 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.496956070302986e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009044962003827095, 'last_eos_top1': 0} +step=246900 lr=6.000000e-04 loss=2.3452 ce=2.3452 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8892459239577875e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009953970089554787, 'last_eos_top1': 0} +step=246950 lr=6.000000e-04 loss=2.6272 ce=2.6272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.582649904070422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008007308468222618, 'last_eos_top1': 0} +step=247000 lr=6.000000e-04 loss=2.6637 ce=2.6637 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1003763574990444e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006182258483022451, 'last_eos_top1': 0} +step=247050 lr=6.000000e-04 loss=2.1498 ce=2.1498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.239590867247898e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00827387347817421, 'last_eos_top1': 0} +step=247100 lr=6.000000e-04 loss=2.4700 ce=2.4700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0661147573264316e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006418459117412567, 'last_eos_top1': 0} +step=247150 lr=6.000000e-04 loss=2.3717 ce=2.3717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2332733604125679e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0054221912287175655, 'last_eos_top1': 0} +step=247200 lr=6.000000e-04 loss=2.2613 ce=2.2613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2474780305637978e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005446859635412693, 'last_eos_top1': 0} +step=247250 lr=6.000000e-04 loss=2.0153 ce=2.0153 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.021476903901203e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065207285806536674, 'last_eos_top1': 0} +step=247300 lr=6.000000e-04 loss=2.1345 ce=2.1345 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.167448529799003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00868287868797779, 'last_eos_top1': 0} +step=247350 lr=6.000000e-04 loss=2.7429 ce=2.7429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3713328371522948e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009550197049975395, 'last_eos_top1': 0} +step=247400 lr=6.000000e-04 loss=2.3210 ce=2.3210 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.468052561103832e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01008931826800108, 'last_eos_top1': 0} +step=247450 lr=6.000000e-04 loss=2.8050 ce=2.8050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0036979801952839e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009078536182641983, 'last_eos_top1': 0} +step=247500 lr=6.000000e-04 loss=2.6301 ce=2.6301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.522347681922838e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013352235779166222, 'last_eos_top1': 0} +step=247550 lr=6.000000e-04 loss=2.5744 ce=2.5744 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7370075511280447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014829830266535282, 'last_eos_top1': 0} +step=247600 lr=6.000000e-04 loss=2.7051 ce=2.7051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.165813829284161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015856152400374413, 'last_eos_top1': 0} +step=247650 lr=6.000000e-04 loss=2.1960 ce=2.1960 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.378059278067667e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007529063615947962, 'last_eos_top1': 0} +step=247700 lr=6.000000e-04 loss=2.5823 ce=2.5823 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2036669431836344e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007851909846067429, 'last_eos_top1': 0} +step=247750 lr=6.000000e-04 loss=2.3098 ce=2.3098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2213066485128365e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005526689812541008, 'last_eos_top1': 0} +step=247800 lr=6.000000e-04 loss=2.2720 ce=2.2720 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.627950253663585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010907082818448544, 'last_eos_top1': 0} +step=247850 lr=6.000000e-04 loss=1.6595 ce=1.6595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4115761587163433e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012576639652252197, 'last_eos_top1': 0} +step=247900 lr=6.000000e-04 loss=2.7297 ce=2.7297 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5873876438708976e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011482940055429935, 'last_eos_top1': 0} +step=247950 lr=6.000000e-04 loss=2.7149 ce=2.7149 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.232323928794358e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009529809467494488, 'last_eos_top1': 0} +step=248000 lr=6.000000e-04 loss=2.3472 ce=2.3472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.881388905341737e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011863088235259056, 'last_eos_top1': 0} +step=248050 lr=6.000000e-04 loss=1.8835 ce=1.8835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.792082553554792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012194689363241196, 'last_eos_top1': 0} +step=248100 lr=6.000000e-04 loss=2.8257 ce=2.8257 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.708286436041817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012202281504869461, 'last_eos_top1': 0} +step=248150 lr=6.000000e-04 loss=2.4329 ce=2.4329 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1978381735389121e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0101973507553339, 'last_eos_top1': 0} +step=248200 lr=6.000000e-04 loss=2.5644 ce=2.5644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.053046258282848e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008092892356216908, 'last_eos_top1': 0} +step=248250 lr=6.000000e-04 loss=2.7432 ce=2.7432 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.557011940662051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011464804410934448, 'last_eos_top1': 0} +step=248300 lr=6.000000e-04 loss=2.7483 ce=2.7483 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.477504833426792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01241525262594223, 'last_eos_top1': 0} +step=248350 lr=6.000000e-04 loss=1.8291 ce=1.8291 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1362910299794748e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015793874859809875, 'last_eos_top1': 0} +step=248400 lr=6.000000e-04 loss=2.4672 ce=2.4672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3752476661466062e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011655613780021667, 'last_eos_top1': 0} +step=248450 lr=6.000000e-04 loss=3.2783 ce=3.2783 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0058245354448445e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0078117153607308865, 'last_eos_top1': 0} +step=248500 lr=6.000000e-04 loss=2.4114 ce=2.4114 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6380850865971297e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006812054198235273, 'last_eos_top1': 0} +step=248550 lr=6.000000e-04 loss=2.4512 ce=2.4512 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.407101707533002e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01027354784309864, 'last_eos_top1': 0} +step=248600 lr=6.000000e-04 loss=3.0582 ce=3.0582 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1048974303994328e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009447416290640831, 'last_eos_top1': 0} +step=248650 lr=6.000000e-04 loss=2.7992 ce=2.7992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5911306036286987e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013047957792878151, 'last_eos_top1': 0} +step=248700 lr=6.000000e-04 loss=2.9791 ce=2.9791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.728849934414029e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016006112098693848, 'last_eos_top1': 0} +step=248750 lr=6.000000e-04 loss=2.2404 ce=2.2404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.085151315142866e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01490018516778946, 'last_eos_top1': 0} +step=248800 lr=6.000000e-04 loss=2.3778 ce=2.3778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0990815098921303e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009620662778615952, 'last_eos_top1': 0} +step=248850 lr=6.000000e-04 loss=2.7802 ce=2.7802 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.360325525951339e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010659405030310154, 'last_eos_top1': 0} +step=248900 lr=6.000000e-04 loss=2.6323 ce=2.6323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.635684258071706e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012760783545672894, 'last_eos_top1': 0} +step=248950 lr=6.000000e-04 loss=2.0666 ce=2.0666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.95391463523265e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011445026844739914, 'last_eos_top1': 0} +step=249000 lr=6.000000e-04 loss=2.4017 ce=2.4017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.170488409727113e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008188771083950996, 'last_eos_top1': 0} +step=249050 lr=6.000000e-04 loss=2.5322 ce=2.5322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.004057574202307e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007100697141140699, 'last_eos_top1': 0} +step=249100 lr=6.000000e-04 loss=2.6917 ce=2.6917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.52900792172295e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011112920939922333, 'last_eos_top1': 0} +step=249150 lr=6.000000e-04 loss=2.6633 ce=2.6633 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.277005584503058e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011126743629574776, 'last_eos_top1': 0} +step=249200 lr=6.000000e-04 loss=2.4377 ce=2.4377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.539349902188405e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012278268113732338, 'last_eos_top1': 0} +step=249250 lr=6.000000e-04 loss=2.1680 ce=2.1680 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4750969462038483e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016448350623250008, 'last_eos_top1': 0} +step=249300 lr=6.000000e-04 loss=2.2135 ce=2.2135 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.020017048693262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017483942210674286, 'last_eos_top1': 0} +step=249350 lr=6.000000e-04 loss=2.6493 ce=2.6493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0299667085055262e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016390569508075714, 'last_eos_top1': 0} +step=249400 lr=6.000000e-04 loss=2.6267 ce=2.6267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.860611731302924e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010592503473162651, 'last_eos_top1': 0} +step=249450 lr=6.000000e-04 loss=1.9508 ce=1.9508 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.070312105701305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010957627557218075, 'last_eos_top1': 0} +[elf_hfds direct rank=1 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4000000 rows=3991809 buffered=8191 part=openwebtext-t5 +step=249500 lr=6.000000e-04 loss=2.8030 ce=2.8030 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4675905731564853e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010437363758683205, 'last_eos_top1': 0} +step=249550 lr=6.000000e-04 loss=2.6392 ce=2.6392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.785811249050312e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0090032909065485, 'last_eos_top1': 0} +step=249600 lr=6.000000e-04 loss=2.3232 ce=2.3232 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0689393093343824e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01044916920363903, 'last_eos_top1': 0} +step=249650 lr=6.000000e-04 loss=2.5397 ce=2.5397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.560624453617493e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012619759887456894, 'last_eos_top1': 0} +step=249700 lr=6.000000e-04 loss=3.6556 ce=3.6556 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0757148629636504e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009188252501189709, 'last_eos_top1': 0} +step=249750 lr=6.000000e-04 loss=2.7748 ce=2.7748 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.912433152843732e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007569311186671257, 'last_eos_top1': 0} +step=249800 lr=6.000000e-04 loss=2.6247 ce=2.6247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.399729606229812e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010865209624171257, 'last_eos_top1': 0} +step=249850 lr=6.000000e-04 loss=2.5148 ce=2.5148 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6765890904935077e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013203082606196404, 'last_eos_top1': 0} +step=249900 lr=6.000000e-04 loss=2.8187 ce=2.8187 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.106823199545033e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015897292643785477, 'last_eos_top1': 0} +step=249950 lr=6.000000e-04 loss=2.6304 ce=2.6304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7309224858763628e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013845417648553848, 'last_eos_top1': 0} +step=250000 lr=6.000000e-04 loss=2.5165 ce=2.5165 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3519729691324756e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013665314763784409, 'last_eos_top1': 0} +step=250050 lr=6.000000e-04 loss=2.7037 ce=2.7037 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8075152183882892e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012108154594898224, 'last_eos_top1': 0} +step=250100 lr=6.000000e-04 loss=2.6941 ce=2.6941 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.510552677558735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011395769193768501, 'last_eos_top1': 0} +step=250150 lr=6.000000e-04 loss=2.9984 ce=2.9984 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.971395810571266e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013029826804995537, 'last_eos_top1': 0} +step=250200 lr=6.000000e-04 loss=2.6843 ce=2.6843 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.87085116119124e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014242475852370262, 'last_eos_top1': 0} +step=250250 lr=6.000000e-04 loss=2.2554 ce=2.2554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.444493358983891e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01766461320221424, 'last_eos_top1': 0} +step=250300 lr=6.000000e-04 loss=2.6955 ce=2.6955 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.403469458542531e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01781485229730606, 'last_eos_top1': 0} +step=250350 lr=6.000000e-04 loss=2.3305 ce=2.3305 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.377343429950997e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012867074459791183, 'last_eos_top1': 0} +step=250400 lr=6.000000e-04 loss=2.7840 ce=2.7840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.7521680244244635e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007980398833751678, 'last_eos_top1': 0} +step=250450 lr=6.000000e-04 loss=2.8864 ce=2.8864 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0060204658657312e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008033689111471176, 'last_eos_top1': 0} +step=250500 lr=6.000000e-04 loss=2.1557 ce=2.1557 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.9650789151201025e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009829340502619743, 'last_eos_top1': 0} +step=250550 lr=6.000000e-04 loss=2.4746 ce=2.4746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.895003596378956e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008356587961316109, 'last_eos_top1': 0} +step=250600 lr=6.000000e-04 loss=2.5219 ce=2.5219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0002575183752924e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007102029863744974, 'last_eos_top1': 0} +step=250650 lr=6.000000e-04 loss=2.2848 ce=2.2848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2788425010512583e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006784248165786266, 'last_eos_top1': 0} +step=250700 lr=6.000000e-04 loss=1.8958 ce=1.8958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4370743883773685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006011759862303734, 'last_eos_top1': 0} +step=250750 lr=6.000000e-04 loss=2.2414 ce=2.2414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2621841708314605e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01223725825548172, 'last_eos_top1': 0} +step=250800 lr=6.000000e-04 loss=2.3727 ce=2.3727 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6365265764761716e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010161066427826881, 'last_eos_top1': 0} +step=250850 lr=6.000000e-04 loss=1.9814 ce=1.9814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.084780685952865e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010494155809283257, 'last_eos_top1': 0} +step=250900 lr=6.000000e-04 loss=1.9110 ce=1.9110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.081027433741838e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009778361767530441, 'last_eos_top1': 0} +step=250950 lr=6.000000e-04 loss=2.9896 ce=2.9896 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3337044038053136e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010138038545846939, 'last_eos_top1': 0} +step=251000 lr=6.000000e-04 loss=2.4619 ce=2.4619 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2083482943126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009241446852684021, 'last_eos_top1': 0} +step=251050 lr=6.000000e-04 loss=2.2017 ce=2.2017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8402719433652237e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012025978416204453, 'last_eos_top1': 0} +step=251100 lr=6.000000e-04 loss=2.0482 ce=2.0482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.814805030648131e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010879571549594402, 'last_eos_top1': 0} +step=251150 lr=6.000000e-04 loss=2.6134 ce=2.6134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.081566369975917e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01274670846760273, 'last_eos_top1': 0} +step=251200 lr=6.000000e-04 loss=2.3515 ce=2.3515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.444805876526516e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01325562596321106, 'last_eos_top1': 0} +step=251250 lr=6.000000e-04 loss=3.6439 ce=3.6439 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2941140084876679e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016411934047937393, 'last_eos_top1': 0} +step=251300 lr=6.000000e-04 loss=2.5870 ce=2.5870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.881461897341069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012203424237668514, 'last_eos_top1': 0} +step=251350 lr=6.000000e-04 loss=2.5761 ce=2.5761 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.115576747513842e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013107231818139553, 'last_eos_top1': 0} +step=251400 lr=6.000000e-04 loss=2.4590 ce=2.4590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0146685427753255e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012596307322382927, 'last_eos_top1': 0} +step=251450 lr=6.000000e-04 loss=2.4023 ce=2.4023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.661508940742351e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010080922394990921, 'last_eos_top1': 0} +step=251500 lr=6.000000e-04 loss=2.6162 ce=2.6162 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.103809063351946e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011866315267980099, 'last_eos_top1': 0} +step=251550 lr=6.000000e-04 loss=1.9978 ce=1.9978 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0729165296652354e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009440025314688683, 'last_eos_top1': 0} +step=251600 lr=6.000000e-04 loss=2.7340 ce=2.7340 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.503686902055051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010249709710478783, 'last_eos_top1': 0} +step=251650 lr=6.000000e-04 loss=2.4205 ce=2.4205 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3926848623668775e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013854975812137127, 'last_eos_top1': 0} +step=251700 lr=6.000000e-04 loss=2.3767 ce=2.3767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.458384400000796e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014715878292918205, 'last_eos_top1': 0} +step=251750 lr=6.000000e-04 loss=2.4068 ce=2.4068 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0214780559181236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01487760804593563, 'last_eos_top1': 0} +step=251800 lr=6.000000e-04 loss=2.9081 ce=2.9081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8972292309626937e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018399789929389954, 'last_eos_top1': 0} +step=251850 lr=6.000000e-04 loss=2.2974 ce=2.2974 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.649130227742717e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019188735634088516, 'last_eos_top1': 0} +step=251900 lr=6.000000e-04 loss=2.7235 ce=2.7235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.125702490564436e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02246861532330513, 'last_eos_top1': 0} +step=251950 lr=6.000000e-04 loss=2.6106 ce=2.6106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.922017852659337e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.018456004559993744, 'last_eos_top1': 0} +step=252000 lr=6.000000e-04 loss=2.8461 ce=2.8461 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.6483898990554735e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01951570063829422, 'last_eos_top1': 0} +step=252050 lr=6.000000e-04 loss=2.9038 ce=2.9038 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4061999536352232e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017399784177541733, 'last_eos_top1': 0} +step=252100 lr=6.000000e-04 loss=2.2051 ce=2.2051 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4688130249851383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01740674301981926, 'last_eos_top1': 0} +step=252150 lr=6.000000e-04 loss=2.7433 ce=2.7433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6793420551694e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01519913412630558, 'last_eos_top1': 0} +step=252200 lr=6.000000e-04 loss=2.7539 ce=2.7539 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0329203986621e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013749522157013416, 'last_eos_top1': 0} +step=252250 lr=6.000000e-04 loss=2.9470 ce=2.9470 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.285438535589492e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017151134088635445, 'last_eos_top1': 0} +step=252300 lr=6.000000e-04 loss=2.3694 ce=2.3694 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.635298864741344e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01675962656736374, 'last_eos_top1': 0} +step=252350 lr=6.000000e-04 loss=2.5821 ce=2.5821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.067712325195316e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017058027908205986, 'last_eos_top1': 0} +step=252400 lr=6.000000e-04 loss=2.4124 ce=2.4124 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.734806109103374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01490947138518095, 'last_eos_top1': 0} +step=252450 lr=6.000000e-04 loss=2.6777 ce=2.6777 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.504860382643528e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.018191048875451088, 'last_eos_top1': 0} +step=252500 lr=6.000000e-04 loss=2.4976 ce=2.4976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.81948040437419e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019653111696243286, 'last_eos_top1': 0} +step=252550 lr=6.000000e-04 loss=2.4840 ce=2.4840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.339126356469933e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01261807233095169, 'last_eos_top1': 0} +step=252600 lr=6.000000e-04 loss=3.0351 ce=3.0351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1213484867766965e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016859449446201324, 'last_eos_top1': 0} +step=252650 lr=6.000000e-04 loss=2.3502 ce=2.3502 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.3860892371158116e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02086333930492401, 'last_eos_top1': 0} +step=252700 lr=6.000000e-04 loss=2.2899 ce=2.2899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4242130419006571e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.022206805646419525, 'last_eos_top1': 0} +step=252750 lr=6.000000e-04 loss=2.4388 ce=2.4388 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2702329513558652e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01654033362865448, 'last_eos_top1': 0} +step=252800 lr=6.000000e-04 loss=2.5105 ce=2.5105 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1881997124874033e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014179235324263573, 'last_eos_top1': 0} +step=252850 lr=6.000000e-04 loss=2.6795 ce=2.6795 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.584302122471854e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01028873398900032, 'last_eos_top1': 0} +step=252900 lr=6.000000e-04 loss=2.3946 ce=2.3946 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3039325494901277e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009386260993778706, 'last_eos_top1': 0} +step=252950 lr=6.000000e-04 loss=2.5677 ce=2.5677 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.187237083096988e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009051903150975704, 'last_eos_top1': 0} +step=253000 lr=6.000000e-04 loss=2.6873 ce=2.6873 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.721357188624097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0063072508201003075, 'last_eos_top1': 0} +step=253050 lr=6.000000e-04 loss=2.6091 ce=2.6091 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.554820513178129e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00859399326145649, 'last_eos_top1': 0} +step=253100 lr=6.000000e-04 loss=2.0715 ce=2.0715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.335444934142288e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00850769318640232, 'last_eos_top1': 0} +step=253150 lr=6.000000e-04 loss=2.1182 ce=2.1182 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.088143658824265e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008773705922067165, 'last_eos_top1': 0} +step=253200 lr=6.000000e-04 loss=2.4474 ce=2.4474 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7647771275951527e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009376592934131622, 'last_eos_top1': 0} +step=253250 lr=6.000000e-04 loss=2.6892 ce=2.6892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.23788502404932e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007790389470756054, 'last_eos_top1': 0} +step=253300 lr=6.000000e-04 loss=2.9414 ce=2.9414 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5877227269811556e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005229879170656204, 'last_eos_top1': 0} +step=253350 lr=6.000000e-04 loss=2.2062 ce=2.2062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.085868441616185e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007681690156459808, 'last_eos_top1': 0} +step=253400 lr=6.000000e-04 loss=2.5666 ce=2.5666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.832594716688618e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005867109168320894, 'last_eos_top1': 0} +step=253450 lr=6.000000e-04 loss=2.3908 ce=2.3908 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.580706561479019e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009009430184960365, 'last_eos_top1': 0} +step=253500 lr=6.000000e-04 loss=2.6889 ce=2.6889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.651351552340202e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006704855710268021, 'last_eos_top1': 0} +step=253550 lr=6.000000e-04 loss=2.3959 ce=2.3959 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.103789590066299e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006801261566579342, 'last_eos_top1': 0} +step=253600 lr=6.000000e-04 loss=2.4613 ce=2.4613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.161566489026882e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007692364044487476, 'last_eos_top1': 0} +step=253650 lr=6.000000e-04 loss=2.5098 ce=2.5098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.572194706677692e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006701840087771416, 'last_eos_top1': 0} +step=253700 lr=6.000000e-04 loss=2.4023 ce=2.4023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.149793186224997e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008269527927041054, 'last_eos_top1': 0} +step=253750 lr=6.000000e-04 loss=2.3570 ce=2.3570 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.369742676615715e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00801185704767704, 'last_eos_top1': 0} +step=253800 lr=6.000000e-04 loss=2.4768 ce=2.4768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.114154762646649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008746695704758167, 'last_eos_top1': 0} +step=253850 lr=6.000000e-04 loss=2.6451 ce=2.6451 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.650711268070154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008191727101802826, 'last_eos_top1': 0} +step=253900 lr=6.000000e-04 loss=3.0267 ce=3.0267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5660809771798085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011147920042276382, 'last_eos_top1': 0} +step=253950 lr=6.000000e-04 loss=2.8436 ce=2.8436 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.2428993007633835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007385013625025749, 'last_eos_top1': 0} +step=254000 lr=6.000000e-04 loss=3.0249 ce=3.0249 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.713096915220376e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008375069126486778, 'last_eos_top1': 0} +step=254050 lr=6.000000e-04 loss=2.4055 ce=2.4055 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.22744584386237e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010953170247375965, 'last_eos_top1': 0} +step=254100 lr=6.000000e-04 loss=2.7919 ce=2.7919 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7882088286569342e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012489024549722672, 'last_eos_top1': 0} +step=254150 lr=6.000000e-04 loss=2.6406 ce=2.6406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2409314877004363e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007459056563675404, 'last_eos_top1': 0} +step=254200 lr=6.000000e-04 loss=2.2855 ce=2.2855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6460973711218685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006976964883506298, 'last_eos_top1': 0} +step=254250 lr=6.000000e-04 loss=2.7438 ce=2.7438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0570160156930797e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058022234588861465, 'last_eos_top1': 0} +step=254300 lr=6.000000e-04 loss=2.1362 ce=2.1362 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.746369424741715e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0050468179397284985, 'last_eos_top1': 0} +step=254350 lr=6.000000e-04 loss=2.5146 ce=2.5146 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.582385812071152e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006123761646449566, 'last_eos_top1': 0} +step=254400 lr=6.000000e-04 loss=3.0921 ce=3.0921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.764162334438879e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007366369478404522, 'last_eos_top1': 0} +step=254450 lr=6.000000e-04 loss=2.9648 ce=2.9648 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3303529158292804e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009034184738993645, 'last_eos_top1': 0} +step=254500 lr=6.000000e-04 loss=2.6794 ce=2.6794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.547128461737884e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007347896695137024, 'last_eos_top1': 0} +step=254550 lr=6.000000e-04 loss=2.6113 ce=2.6113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1128927326353732e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005374397151172161, 'last_eos_top1': 0} +step=254600 lr=6.000000e-04 loss=2.1154 ce=2.1154 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.233125870698132e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023501371033489704, 'last_eos_top1': 0} +step=254650 lr=6.000000e-04 loss=2.4689 ce=2.4689 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.168249400507193e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00388708827085793, 'last_eos_top1': 0} +step=254700 lr=6.000000e-04 loss=2.1200 ce=2.1200 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.192503289843444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0048159281723201275, 'last_eos_top1': 0} +step=254750 lr=6.000000e-04 loss=2.8440 ce=2.8440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5989102141465992e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005633811000734568, 'last_eos_top1': 0} +step=254800 lr=6.000000e-04 loss=2.4989 ce=2.4989 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0272455256199464e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065300967544317245, 'last_eos_top1': 0} +step=254850 lr=6.000000e-04 loss=2.2186 ce=2.2186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1415801054681651e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007663242984563112, 'last_eos_top1': 0} +step=254900 lr=6.000000e-04 loss=2.2245 ce=2.2245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.106379204429686e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008803563192486763, 'last_eos_top1': 0} +step=254950 lr=6.000000e-04 loss=2.6992 ce=2.6992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8769886082736775e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009778337553143501, 'last_eos_top1': 0} +step=255000 lr=6.000000e-04 loss=2.7988 ce=2.7988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.744696863985155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016065305098891258, 'last_eos_top1': 0} +step=255050 lr=6.000000e-04 loss=2.3692 ce=2.3692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1843632819363847e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018038131296634674, 'last_eos_top1': 0} +step=255100 lr=6.000000e-04 loss=2.4839 ce=2.4839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5175626685959287e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015962785109877586, 'last_eos_top1': 0} +step=255150 lr=6.000000e-04 loss=2.5096 ce=2.5096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3855196559452452e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006473243236541748, 'last_eos_top1': 0} +step=255200 lr=6.000000e-04 loss=2.1366 ce=2.1366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1495601938804612e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058759646490216255, 'last_eos_top1': 0} +step=255250 lr=6.000000e-04 loss=2.6065 ce=2.6065 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.65109871281311e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008249327540397644, 'last_eos_top1': 0} +step=255300 lr=6.000000e-04 loss=2.6234 ce=2.6234 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.982438314182218e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009536944329738617, 'last_eos_top1': 0} +step=255350 lr=6.000000e-04 loss=2.5713 ce=2.5713 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7496509826742113e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00951949879527092, 'last_eos_top1': 0} +step=255400 lr=6.000000e-04 loss=3.0262 ce=3.0262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0284234122082125e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010895458050072193, 'last_eos_top1': 0} +step=255450 lr=6.000000e-04 loss=1.8708 ce=1.8708 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3121392839821056e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007108454592525959, 'last_eos_top1': 0} +step=255500 lr=6.000000e-04 loss=3.0626 ce=3.0626 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.2564537327270955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007143063470721245, 'last_eos_top1': 0} +step=255550 lr=6.000000e-04 loss=2.4640 ce=2.4640 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0500530152057763e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009058862924575806, 'last_eos_top1': 0} +step=255600 lr=6.000000e-04 loss=2.7883 ce=2.7883 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.742188477597665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009501151740550995, 'last_eos_top1': 0} +step=255650 lr=6.000000e-04 loss=2.6741 ce=2.6741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.873584763729014e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008156104013323784, 'last_eos_top1': 0} +step=255700 lr=6.000000e-04 loss=2.8602 ce=2.8602 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1561518476810306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007670189253985882, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=3 worker=1] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=5 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4100000 rows=4091809 buffered=8191 part=openwebtext-t5 +step=255750 lr=6.000000e-04 loss=2.4165 ce=2.4165 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9367744243936613e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007843850180506706, 'last_eos_top1': 0} +step=255800 lr=6.000000e-04 loss=2.2041 ce=2.2041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.010934496065602e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006747791543602943, 'last_eos_top1': 0} +step=255850 lr=6.000000e-04 loss=2.3262 ce=2.3262 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2914088074467145e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007981294766068459, 'last_eos_top1': 0} +step=255900 lr=6.000000e-04 loss=2.9247 ce=2.9247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.356518264918122e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009546052664518356, 'last_eos_top1': 0} +step=255950 lr=6.000000e-04 loss=1.8642 ce=1.8642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.575378882989753e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010158943012356758, 'last_eos_top1': 0} +step=256000 lr=6.000000e-04 loss=2.3728 ce=2.3728 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6855115973157808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01016067061573267, 'last_eos_top1': 0} +step=256050 lr=6.000000e-04 loss=2.2173 ce=2.2173 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5266134141711518e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011705294251441956, 'last_eos_top1': 0} +step=256100 lr=6.000000e-04 loss=2.2934 ce=2.2934 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1692963198584039e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01191919669508934, 'last_eos_top1': 0} +step=256150 lr=6.000000e-04 loss=1.9703 ce=1.9703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.051824923109962e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0107191801071167, 'last_eos_top1': 0} +step=256200 lr=6.000000e-04 loss=2.5442 ce=2.5442 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1047092812077608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010456554591655731, 'last_eos_top1': 0} +step=256250 lr=6.000000e-04 loss=2.6610 ce=2.6610 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0368512196000665e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010978087782859802, 'last_eos_top1': 0} +step=256300 lr=6.000000e-04 loss=2.4296 ce=2.4296 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7307115639559925e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013294784352183342, 'last_eos_top1': 0} +step=256350 lr=6.000000e-04 loss=2.7822 ce=2.7822 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.590751465613721e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015212584286928177, 'last_eos_top1': 0} +step=256400 lr=6.000000e-04 loss=2.2729 ce=2.2729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.099153779359767e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010691246017813683, 'last_eos_top1': 0} +step=256450 lr=6.000000e-04 loss=2.1952 ce=2.1952 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.122016879497096e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014072593301534653, 'last_eos_top1': 0} +step=256500 lr=6.000000e-04 loss=2.6558 ce=2.6558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.075839675962925e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016110774129629135, 'last_eos_top1': 0} +step=256550 lr=6.000000e-04 loss=2.5688 ce=2.5688 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3006447261432186e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01389254815876484, 'last_eos_top1': 0} +step=256600 lr=6.000000e-04 loss=2.1541 ce=2.1541 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.9388867157395e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014543730765581131, 'last_eos_top1': 0} +step=256650 lr=6.000000e-04 loss=1.9308 ce=1.9308 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.45500323723536e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015296917408704758, 'last_eos_top1': 0} +step=256700 lr=6.000000e-04 loss=2.3553 ce=2.3553 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8659542547538877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015790246427059174, 'last_eos_top1': 0} +step=256750 lr=6.000000e-04 loss=2.1492 ce=2.1492 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4967986317060422e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01190892793238163, 'last_eos_top1': 0} +step=256800 lr=6.000000e-04 loss=2.5172 ce=2.5172 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.968354788725264e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010744942352175713, 'last_eos_top1': 0} +step=256850 lr=6.000000e-04 loss=2.1245 ce=2.1245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4670260497950949e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014422724023461342, 'last_eos_top1': 0} +step=256900 lr=6.000000e-04 loss=2.2416 ce=2.2416 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1583570085349493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01107779797166586, 'last_eos_top1': 0} +step=256950 lr=6.000000e-04 loss=2.4772 ce=2.4772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.095891982724424e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013441028073430061, 'last_eos_top1': 0} +step=257000 lr=6.000000e-04 loss=2.6186 ce=2.6186 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.221067789942026e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012792358174920082, 'last_eos_top1': 0} +step=257050 lr=6.000000e-04 loss=2.4521 ce=2.4521 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.967583092569839e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011184325441718102, 'last_eos_top1': 0} +step=257100 lr=6.000000e-04 loss=2.0571 ce=2.0571 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4144308806862682e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00724202673882246, 'last_eos_top1': 0} +step=257150 lr=6.000000e-04 loss=2.8563 ce=2.8563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.331568096764386e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009451424703001976, 'last_eos_top1': 0} +step=257200 lr=6.000000e-04 loss=2.5043 ce=2.5043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.4796238449198427e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007365914061665535, 'last_eos_top1': 0} +step=257250 lr=6.000000e-04 loss=2.7021 ce=2.7021 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.657397873117588e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011738007888197899, 'last_eos_top1': 0} +step=257300 lr=6.000000e-04 loss=2.7977 ce=2.7977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.978255209513009e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015103236772119999, 'last_eos_top1': 0} +step=257350 lr=6.000000e-04 loss=2.5868 ce=2.5868 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.824464435339905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015595894306898117, 'last_eos_top1': 0} +step=257400 lr=6.000000e-04 loss=2.1718 ce=2.1718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2218926713103428e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014519416727125645, 'last_eos_top1': 0} +step=257450 lr=6.000000e-04 loss=2.6922 ce=2.6922 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1463998816907406e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012331554666161537, 'last_eos_top1': 0} +step=257500 lr=6.000000e-04 loss=2.2700 ce=2.2700 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.329029358515982e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009783453308045864, 'last_eos_top1': 0} +step=257550 lr=6.000000e-04 loss=2.7015 ce=2.7015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1344456652295776e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011605610139667988, 'last_eos_top1': 0} +step=257600 lr=6.000000e-04 loss=2.4117 ce=2.4117 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.368535418412648e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014581133611500263, 'last_eos_top1': 0} +step=257650 lr=6.000000e-04 loss=2.7802 ce=2.7802 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.420493370271288e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014546283520758152, 'last_eos_top1': 0} +step=257700 lr=6.000000e-04 loss=2.4594 ce=2.4594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.091003287001513e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017008308321237564, 'last_eos_top1': 0} +step=257750 lr=6.000000e-04 loss=2.5302 ce=2.5302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4929075405234471e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008305744268000126, 'last_eos_top1': 0} +step=257800 lr=6.000000e-04 loss=2.5236 ce=2.5236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.2430252506164834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005526621825993061, 'last_eos_top1': 0} +step=257850 lr=6.000000e-04 loss=2.0066 ce=2.0066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.819309181883e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005337283946573734, 'last_eos_top1': 0} +step=257900 lr=6.000000e-04 loss=2.5141 ce=2.5141 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0491662578715477e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009470236487686634, 'last_eos_top1': 0} +step=257950 lr=6.000000e-04 loss=1.8469 ce=1.8469 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.458985005039722e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00923236459493637, 'last_eos_top1': 0} +step=258000 lr=6.000000e-04 loss=2.7634 ce=2.7634 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1536565327551216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008658254519104958, 'last_eos_top1': 0} +step=258050 lr=6.000000e-04 loss=2.1160 ce=2.1160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.0772154029109515e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0069886380806565285, 'last_eos_top1': 0} +step=258100 lr=6.000000e-04 loss=2.5648 ce=2.5648 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3517324987333268e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007261272054165602, 'last_eos_top1': 0} +step=258150 lr=6.000000e-04 loss=2.2078 ce=2.2078 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.026026984618511e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012448901310563087, 'last_eos_top1': 0} +step=258200 lr=6.000000e-04 loss=2.6475 ce=2.6475 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2620892448467202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014901943504810333, 'last_eos_top1': 0} +step=258250 lr=6.000000e-04 loss=2.5097 ce=2.5097 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.550951522716787e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013208482414484024, 'last_eos_top1': 0} +step=258300 lr=6.000000e-04 loss=2.2360 ce=2.2360 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.571114808029961e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012287216261029243, 'last_eos_top1': 0} +step=258350 lr=6.000000e-04 loss=2.3609 ce=2.3609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5196001186268404e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02139042690396309, 'last_eos_top1': 0} +step=258400 lr=6.000000e-04 loss=2.6044 ce=2.6044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.530554254713934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.020791374146938324, 'last_eos_top1': 0} +step=258450 lr=6.000000e-04 loss=2.6333 ce=2.6333 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5493176508462057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018087364733219147, 'last_eos_top1': 0} +step=258500 lr=6.000000e-04 loss=2.5805 ce=2.5805 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.654793757363223e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013395641930401325, 'last_eos_top1': 0} +step=258550 lr=6.000000e-04 loss=2.6935 ce=2.6935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9286328097223304e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013370628468692303, 'last_eos_top1': 0} +step=258600 lr=6.000000e-04 loss=1.7920 ce=1.7920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1706702935043722e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00986488163471222, 'last_eos_top1': 0} +step=258650 lr=6.000000e-04 loss=2.2206 ce=2.2206 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2852653526351787e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00933682732284069, 'last_eos_top1': 0} +step=258700 lr=6.000000e-04 loss=2.7347 ce=2.7347 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.376169494615169e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008938232436776161, 'last_eos_top1': 0} +step=258750 lr=6.000000e-04 loss=2.2693 ce=2.2693 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1018046027165838e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0067850095219910145, 'last_eos_top1': 0} +step=258800 lr=6.000000e-04 loss=2.5837 ce=2.5837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2250158255919814e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003929490223526955, 'last_eos_top1': 0} +step=258850 lr=6.000000e-04 loss=2.4344 ce=2.4344 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.019528988574166e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.002781572751700878, 'last_eos_top1': 0} +step=258900 lr=6.000000e-04 loss=2.1309 ce=2.1309 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.5389443736639805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0061354245990514755, 'last_eos_top1': 0} +step=258950 lr=6.000000e-04 loss=2.8758 ce=2.8758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3765800758847035e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008093461394309998, 'last_eos_top1': 0} +step=259000 lr=6.000000e-04 loss=2.2317 ce=2.2317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.69087197718909e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009953455999493599, 'last_eos_top1': 0} +step=259050 lr=6.000000e-04 loss=2.4914 ce=2.4914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4690418538521044e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007995610125362873, 'last_eos_top1': 0} +step=259100 lr=6.000000e-04 loss=2.4047 ce=2.4047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4974184523453005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00757612194865942, 'last_eos_top1': 0} +step=259150 lr=6.000000e-04 loss=2.7489 ce=2.7489 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6333415260305628e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006856611929833889, 'last_eos_top1': 0} +step=259200 lr=6.000000e-04 loss=2.7218 ce=2.7218 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2890124708064832e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0052212439477443695, 'last_eos_top1': 0} +step=259250 lr=6.000000e-04 loss=2.5857 ce=2.5857 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2498671821958851e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006912998855113983, 'last_eos_top1': 0} +step=259300 lr=6.000000e-04 loss=2.8528 ce=2.8528 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1789263226091862e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00423472560942173, 'last_eos_top1': 0} +step=259350 lr=6.000000e-04 loss=2.5393 ce=2.5393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.550751201459207e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005741599015891552, 'last_eos_top1': 0} +step=259400 lr=6.000000e-04 loss=2.6818 ce=2.6818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.949737432179973e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0046345023438334465, 'last_eos_top1': 0} +step=259450 lr=6.000000e-04 loss=2.7538 ce=2.7538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2154800060670823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007931031286716461, 'last_eos_top1': 0} +step=259500 lr=6.000000e-04 loss=2.8096 ce=2.8096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.579330940643558e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010156424716114998, 'last_eos_top1': 0} +step=259550 lr=6.000000e-04 loss=2.2827 ce=2.2827 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.534188742080005e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010689850896596909, 'last_eos_top1': 0} +step=259600 lr=6.000000e-04 loss=2.9515 ce=2.9515 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.948537470132578e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008931663818657398, 'last_eos_top1': 0} +step=259650 lr=6.000000e-04 loss=2.8058 ce=2.8058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2112848708056845e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008142423816025257, 'last_eos_top1': 0} +step=259700 lr=6.000000e-04 loss=2.2849 ce=2.2849 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0242241842206568e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058351727202534676, 'last_eos_top1': 0} +step=259750 lr=6.000000e-04 loss=2.3897 ce=2.3897 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2637661711778492e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005209094844758511, 'last_eos_top1': 0} +step=259800 lr=6.000000e-04 loss=2.3668 ce=2.3668 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1383116543584038e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0049998946487903595, 'last_eos_top1': 0} +step=259850 lr=6.000000e-04 loss=2.3432 ce=2.3432 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6986428818199784e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00608763238415122, 'last_eos_top1': 0} +step=259900 lr=6.000000e-04 loss=2.2931 ce=2.2931 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1149837266420946e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0051219407469034195, 'last_eos_top1': 0} +step=259950 lr=6.000000e-04 loss=2.2997 ce=2.2997 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4636336345574819e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003496126737445593, 'last_eos_top1': 0} +step=260000 lr=6.000000e-04 loss=1.8338 ce=1.8338 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0960555300698616e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00412222184240818, 'last_eos_top1': 0} +step=260050 lr=6.000000e-04 loss=2.2631 ce=2.2631 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8575643480289727e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004961368627846241, 'last_eos_top1': 0} +step=260100 lr=6.000000e-04 loss=2.3870 ce=2.3870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.43926297116559e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005771740339696407, 'last_eos_top1': 0} +step=260150 lr=6.000000e-04 loss=2.2365 ce=2.2365 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6313781088683754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006628952920436859, 'last_eos_top1': 0} +step=260200 lr=6.000000e-04 loss=2.4129 ce=2.4129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7689304513623938e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007033597212284803, 'last_eos_top1': 0} +step=260250 lr=6.000000e-04 loss=1.9060 ce=1.9060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6187550500035286e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006327959708869457, 'last_eos_top1': 0} +step=260300 lr=6.000000e-04 loss=2.1235 ce=2.1235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0884395780740306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006194128654897213, 'last_eos_top1': 0} +step=260350 lr=6.000000e-04 loss=2.6136 ce=2.6136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5875722965574823e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0034670569002628326, 'last_eos_top1': 0} +step=260400 lr=6.000000e-04 loss=2.4766 ce=2.4766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0638535059115384e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002821840811520815, 'last_eos_top1': 0} +step=260450 lr=6.000000e-04 loss=2.5105 ce=2.5105 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6804999177111313e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003071631072089076, 'last_eos_top1': 0} +step=260500 lr=6.000000e-04 loss=2.4306 ce=2.4306 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0152142749575432e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0024946515914052725, 'last_eos_top1': 0} +step=260550 lr=6.000000e-04 loss=2.7424 ce=2.7424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.302828064188361e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0026270318776369095, 'last_eos_top1': 0} +step=260600 lr=6.000000e-04 loss=2.3367 ce=2.3367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.274589158128947e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0018690378637984395, 'last_eos_top1': 0} +step=260650 lr=6.000000e-04 loss=2.9069 ce=2.9069 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8540973542258143e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0023258537985384464, 'last_eos_top1': 0} +step=260700 lr=6.000000e-04 loss=1.9527 ce=1.9527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1557872716803104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003811896312981844, 'last_eos_top1': 0} +step=260750 lr=6.000000e-04 loss=2.3429 ce=2.3429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.80045354278991e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003756217658519745, 'last_eos_top1': 0} +step=260800 lr=6.000000e-04 loss=2.7954 ce=2.7954 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1477175323525444e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004301651380956173, 'last_eos_top1': 0} +step=260850 lr=6.000000e-04 loss=2.4595 ce=2.4595 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.95321136340499e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004241716582328081, 'last_eos_top1': 0} +step=260900 lr=6.000000e-04 loss=2.0900 ce=2.0900 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.8315538656897843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00506383553147316, 'last_eos_top1': 0} +step=260950 lr=6.000000e-04 loss=2.6288 ce=2.6288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7042202671291307e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004581236280500889, 'last_eos_top1': 0} +step=261000 lr=6.000000e-04 loss=2.2020 ce=2.2020 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.413704224745743e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006284581497311592, 'last_eos_top1': 0} +step=261050 lr=6.000000e-04 loss=2.1887 ce=2.1887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.01839780958835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007053001783788204, 'last_eos_top1': 0} +step=261100 lr=6.000000e-04 loss=2.5387 ce=2.5387 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.379125236184336e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0071634938940405846, 'last_eos_top1': 0} +step=261150 lr=6.000000e-04 loss=2.6453 ce=2.6453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.772016710485332e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0064372955821454525, 'last_eos_top1': 0} +step=261200 lr=6.000000e-04 loss=2.9372 ce=2.9372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1139308955753222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0063524190336465836, 'last_eos_top1': 0} +step=261250 lr=6.000000e-04 loss=2.8386 ce=2.8386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7781370843295008e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00775420106947422, 'last_eos_top1': 0} +step=261300 lr=6.000000e-04 loss=2.6235 ce=2.6235 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.606788282515481e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007879825308918953, 'last_eos_top1': 0} +step=261350 lr=6.000000e-04 loss=2.5635 ce=2.5635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.636404704186134e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00761396624147892, 'last_eos_top1': 0} +step=261400 lr=6.000000e-04 loss=2.2450 ce=2.2450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.990774217119906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007613667286932468, 'last_eos_top1': 0} +step=261450 lr=6.000000e-04 loss=2.1207 ce=2.1207 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0091645637876354e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008207379840314388, 'last_eos_top1': 0} +step=261500 lr=6.000000e-04 loss=2.3609 ce=2.3609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.369002105406253e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010906894691288471, 'last_eos_top1': 0} +step=261550 lr=6.000000e-04 loss=3.1611 ce=3.1611 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1499541869852692e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011693020351231098, 'last_eos_top1': 0} +step=261600 lr=6.000000e-04 loss=2.6367 ce=2.6367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0683002983569168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008750936016440392, 'last_eos_top1': 0} +step=261650 lr=6.000000e-04 loss=2.7962 ce=2.7962 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2036550288030412e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00689466018229723, 'last_eos_top1': 0} +step=261700 lr=6.000000e-04 loss=2.2375 ce=2.2375 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.250185141543625e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005932901054620743, 'last_eos_top1': 0} +step=261750 lr=6.000000e-04 loss=2.3798 ce=2.3798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9451843400020152e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008136020973324776, 'last_eos_top1': 0} +step=261800 lr=6.000000e-04 loss=2.4850 ce=2.4850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.803578925319016e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009866040199995041, 'last_eos_top1': 0} +step=261850 lr=6.000000e-04 loss=2.3657 ce=2.3657 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1452295439084992e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0074887340888381, 'last_eos_top1': 0} +step=261900 lr=6.000000e-04 loss=2.8386 ce=2.8386 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6474608603166416e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00934200081974268, 'last_eos_top1': 0} +step=261950 lr=6.000000e-04 loss=2.5985 ce=2.5985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.208908184315078e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00868326798081398, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4200000 rows=4191809 buffered=8191 part=openwebtext-t5 +step=262000 lr=6.000000e-04 loss=1.9544 ce=1.9544 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.253584721591324e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012646188959479332, 'last_eos_top1': 0} +step=262050 lr=6.000000e-04 loss=2.4593 ce=2.4593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.94377023744164e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008871889673173428, 'last_eos_top1': 0} +step=262100 lr=6.000000e-04 loss=2.1860 ce=2.1860 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0925185051746666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00599268265068531, 'last_eos_top1': 0} +step=262150 lr=6.000000e-04 loss=2.5363 ce=2.5363 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.129969839705154e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0048513393849134445, 'last_eos_top1': 0} +step=262200 lr=6.000000e-04 loss=2.3812 ce=2.3812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3923086700960994e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004428453277796507, 'last_eos_top1': 0} +step=262250 lr=6.000000e-04 loss=2.1465 ce=2.1465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2467708074836992e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004996907897293568, 'last_eos_top1': 0} +step=262300 lr=6.000000e-04 loss=2.4392 ce=2.4392 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6922593204071745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002617879305034876, 'last_eos_top1': 0} +step=262350 lr=6.000000e-04 loss=2.5080 ce=2.5080 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5051547052280512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002392503200098872, 'last_eos_top1': 0} +step=262400 lr=6.000000e-04 loss=2.0780 ce=2.0780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6345504263881594e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004279156215488911, 'last_eos_top1': 0} +step=262450 lr=6.000000e-04 loss=2.6041 ce=2.6041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4073208149056882e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003672079648822546, 'last_eos_top1': 0} +step=262500 lr=6.000000e-04 loss=2.4676 ce=2.4676 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.732132194680162e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004337802994996309, 'last_eos_top1': 0} +step=262550 lr=6.000000e-04 loss=2.2867 ce=2.2867 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2326825526542962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004341048188507557, 'last_eos_top1': 0} +step=262600 lr=6.000000e-04 loss=3.0995 ce=3.0995 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.447221262031235e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005280961748212576, 'last_eos_top1': 0} +step=262650 lr=6.000000e-04 loss=2.4567 ce=2.4567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5503359463764355e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003319896524772048, 'last_eos_top1': 0} +step=262700 lr=6.000000e-04 loss=1.8530 ce=1.8530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.916161277738865e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004425272345542908, 'last_eos_top1': 0} +step=262750 lr=6.000000e-04 loss=2.2067 ce=2.2067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.2341435952694155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0036679634358733892, 'last_eos_top1': 0} +step=262800 lr=6.000000e-04 loss=2.7969 ce=2.7969 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2043209608236793e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004718794487416744, 'last_eos_top1': 0} +step=262850 lr=6.000000e-04 loss=2.1022 ce=2.1022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2126218280172907e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005829969421029091, 'last_eos_top1': 0} +step=262900 lr=6.000000e-04 loss=2.2062 ce=2.2062 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.747402347973548e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004047351889312267, 'last_eos_top1': 0} +step=262950 lr=6.000000e-04 loss=2.2433 ce=2.2433 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3398119335761294e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0048040361143648624, 'last_eos_top1': 0} +step=263000 lr=6.000000e-04 loss=2.7525 ce=2.7525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.187934296671301e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00504491850733757, 'last_eos_top1': 0} +step=263050 lr=6.000000e-04 loss=2.2746 ce=2.2746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.064074215013534e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004500021226704121, 'last_eos_top1': 0} +step=263100 lr=6.000000e-04 loss=2.5558 ce=2.5558 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.8241425247397274e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00493268296122551, 'last_eos_top1': 0} +step=263150 lr=6.000000e-04 loss=2.5043 ce=2.5043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.851843631418888e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003792176954448223, 'last_eos_top1': 0} +step=263200 lr=6.000000e-04 loss=2.7424 ce=2.7424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.550078534113709e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004798112902790308, 'last_eos_top1': 0} +step=263250 lr=6.000000e-04 loss=2.3134 ce=2.3134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.813142625906039e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0052869683131575584, 'last_eos_top1': 0} +step=263300 lr=6.000000e-04 loss=2.6580 ce=2.6580 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.840415279730223e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006233766674995422, 'last_eos_top1': 0} +step=263350 lr=6.000000e-04 loss=2.1304 ce=2.1304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.151055923604872e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0059345681220293045, 'last_eos_top1': 0} +step=263400 lr=6.000000e-04 loss=2.7199 ce=2.7199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2598131181439385e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0073462314903736115, 'last_eos_top1': 0} +step=263450 lr=6.000000e-04 loss=2.9955 ce=2.9955 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4232236935640685e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006790464743971825, 'last_eos_top1': 0} +step=263500 lr=6.000000e-04 loss=2.5529 ce=2.5529 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1457455912022851e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006616316735744476, 'last_eos_top1': 0} +step=263550 lr=6.000000e-04 loss=2.4698 ce=2.4698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5563564122421667e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005812777206301689, 'last_eos_top1': 0} +step=263600 lr=6.000000e-04 loss=2.6128 ce=2.6128 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.805387677857652e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008480645716190338, 'last_eos_top1': 0} +step=263650 lr=6.000000e-04 loss=3.1879 ce=3.1879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0571900677168742e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006571687757968903, 'last_eos_top1': 0} +step=263700 lr=6.000000e-04 loss=2.6453 ce=2.6453 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.267419404233806e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005750059150159359, 'last_eos_top1': 0} +step=263750 lr=6.000000e-04 loss=2.5216 ce=2.5216 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0982961612171493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007259721402078867, 'last_eos_top1': 0} +step=263800 lr=6.000000e-04 loss=2.3644 ce=2.3644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.3910561038937885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006210430525243282, 'last_eos_top1': 0} +step=263850 lr=6.000000e-04 loss=2.5912 ce=2.5912 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.036276989907492e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004740257281810045, 'last_eos_top1': 0} +step=263900 lr=6.000000e-04 loss=2.0293 ce=2.0293 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1073421774199232e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005258636549115181, 'last_eos_top1': 0} +step=263950 lr=6.000000e-04 loss=2.7369 ce=2.7369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2060187145834789e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005114962346851826, 'last_eos_top1': 0} +step=264000 lr=6.000000e-04 loss=2.3990 ce=2.3990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2050715668010525e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005549568682909012, 'last_eos_top1': 0} +step=264050 lr=6.000000e-04 loss=2.6389 ce=2.6389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.146773896238301e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065886490046978, 'last_eos_top1': 0} +step=264100 lr=6.000000e-04 loss=2.1839 ce=2.1839 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.486596925649792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006465991958975792, 'last_eos_top1': 0} +step=264150 lr=6.000000e-04 loss=1.8266 ce=1.8266 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6068086551968008e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005517445970326662, 'last_eos_top1': 0} +step=264200 lr=6.000000e-04 loss=2.5593 ce=2.5593 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.478987127891742e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009169114753603935, 'last_eos_top1': 0} +step=264250 lr=6.000000e-04 loss=2.9473 ce=2.9473 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.481360934325494e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0068526491522789, 'last_eos_top1': 0} +step=264300 lr=6.000000e-04 loss=3.0290 ce=3.0290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.899359813251067e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007603498175740242, 'last_eos_top1': 0} +step=264350 lr=6.000000e-04 loss=2.9567 ce=2.9567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0827796359080821e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007288055494427681, 'last_eos_top1': 0} +step=264400 lr=6.000000e-04 loss=2.7692 ce=2.7692 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.944857093709288e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007182097993791103, 'last_eos_top1': 0} +step=264450 lr=6.000000e-04 loss=2.1174 ce=2.1174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0156452844967134e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010808621533215046, 'last_eos_top1': 0} +step=264500 lr=6.000000e-04 loss=2.7012 ce=2.7012 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9210481696063653e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008820058777928352, 'last_eos_top1': 0} +step=264550 lr=6.000000e-04 loss=2.2876 ce=2.2876 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.51155618368648e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0068852948024868965, 'last_eos_top1': 0} +step=264600 lr=6.000000e-04 loss=2.3462 ce=2.3462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.6293927122605965e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008070574142038822, 'last_eos_top1': 0} +step=264650 lr=6.000000e-04 loss=2.4587 ce=2.4587 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2026927834085654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005788179114460945, 'last_eos_top1': 0} +step=264700 lr=6.000000e-04 loss=2.7267 ce=2.7267 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.039569365791976e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003660631598904729, 'last_eos_top1': 0} +step=264750 lr=6.000000e-04 loss=2.5762 ce=2.5762 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.455440259538591e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006017849314957857, 'last_eos_top1': 0} +step=264800 lr=6.000000e-04 loss=2.8536 ce=2.8536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.277680323459208e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058088842779397964, 'last_eos_top1': 0} +step=264850 lr=6.000000e-04 loss=1.9918 ce=1.9918 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.414278449374251e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005307436920702457, 'last_eos_top1': 0} +step=264900 lr=6.000000e-04 loss=2.0788 ce=2.0788 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.667459582677111e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006190536543726921, 'last_eos_top1': 0} +step=264950 lr=6.000000e-04 loss=2.4254 ce=2.4254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.63611717754975e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006511686369776726, 'last_eos_top1': 0} +step=265000 lr=6.000000e-04 loss=2.4395 ce=2.4395 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.182505366159603e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007027375511825085, 'last_eos_top1': 0} +step=265050 lr=6.000000e-04 loss=2.6438 ce=2.6438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4904115232639015e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00549855874851346, 'last_eos_top1': 0} +step=265100 lr=6.000000e-04 loss=2.9378 ce=2.9378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7349189874948934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00732188206166029, 'last_eos_top1': 0} +step=265150 lr=6.000000e-04 loss=2.4933 ce=2.4933 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1081803677370772e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005781154613941908, 'last_eos_top1': 0} +step=265200 lr=6.000000e-04 loss=2.4404 ce=2.4404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9556729714386165e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005658139940351248, 'last_eos_top1': 0} +step=265250 lr=6.000000e-04 loss=2.3665 ce=2.3665 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.027078517130576e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0075708357617259026, 'last_eos_top1': 0} +step=265300 lr=6.000000e-04 loss=2.9758 ce=2.9758 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0253035725327209e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007785359863191843, 'last_eos_top1': 0} +step=265350 lr=6.000000e-04 loss=2.0467 ce=2.0467 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.306649083853699e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006480209529399872, 'last_eos_top1': 0} +step=265400 lr=6.000000e-04 loss=2.4029 ce=2.4029 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.485006451839581e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007805565372109413, 'last_eos_top1': 0} +step=265450 lr=6.000000e-04 loss=2.3428 ce=2.3428 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4273700799094513e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009077461436390877, 'last_eos_top1': 0} +step=265500 lr=6.000000e-04 loss=2.5033 ce=2.5033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.240329584106803e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008541077375411987, 'last_eos_top1': 0} +step=265550 lr=6.000000e-04 loss=2.3967 ce=2.3967 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.01671001984505e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01078037079423666, 'last_eos_top1': 0} +step=265600 lr=6.000000e-04 loss=2.7268 ce=2.7268 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1114609454525635e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014831176958978176, 'last_eos_top1': 0} +step=265650 lr=6.000000e-04 loss=2.5990 ce=2.5990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.753498382749967e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015733256936073303, 'last_eos_top1': 0} +step=265700 lr=6.000000e-04 loss=2.2745 ce=2.2745 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2290138329262845e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01344582810997963, 'last_eos_top1': 0} +step=265750 lr=6.000000e-04 loss=2.2734 ce=2.2734 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.251141257962445e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01238488033413887, 'last_eos_top1': 0} +step=265800 lr=6.000000e-04 loss=2.3142 ce=2.3142 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.73426472733263e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010104147717356682, 'last_eos_top1': 0} +step=265850 lr=6.000000e-04 loss=2.5895 ce=2.5895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5053114111651666e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007992965169250965, 'last_eos_top1': 0} +step=265900 lr=6.000000e-04 loss=2.4361 ce=2.4361 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6710200725356117e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00920822098851204, 'last_eos_top1': 0} +step=265950 lr=6.000000e-04 loss=2.9348 ce=2.9348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5074438124429435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009966202080249786, 'last_eos_top1': 0} +step=266000 lr=6.000000e-04 loss=2.3522 ce=2.3522 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.92268707905896e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009903833270072937, 'last_eos_top1': 0} +step=266050 lr=6.000000e-04 loss=2.8314 ce=2.8314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.097064250643598e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013497368432581425, 'last_eos_top1': 0} +step=266100 lr=6.000000e-04 loss=2.2305 ce=2.2305 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.070826773589943e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012508345767855644, 'last_eos_top1': 0} +step=266150 lr=6.000000e-04 loss=2.9286 ce=2.9286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.848903806007002e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014401517808437347, 'last_eos_top1': 0} +step=266200 lr=6.000000e-04 loss=3.1897 ce=3.1897 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6472105673747137e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014705678448081017, 'last_eos_top1': 0} +step=266250 lr=6.000000e-04 loss=2.3812 ce=2.3812 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.652015251049306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014233741909265518, 'last_eos_top1': 0} +step=266300 lr=6.000000e-04 loss=2.2510 ce=2.2510 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.111784513341263e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010139583609998226, 'last_eos_top1': 0} +step=266350 lr=6.000000e-04 loss=2.4784 ce=2.4784 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1522854038048536e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008467629551887512, 'last_eos_top1': 0} +step=266400 lr=6.000000e-04 loss=2.3352 ce=2.3352 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6713554689195007e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009249480441212654, 'last_eos_top1': 0} +step=266450 lr=6.000000e-04 loss=1.6578 ce=1.6578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.073539846809581e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01237943023443222, 'last_eos_top1': 0} +step=266500 lr=6.000000e-04 loss=2.1717 ce=2.1717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2390156371111516e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009104082360863686, 'last_eos_top1': 0} +step=266550 lr=6.000000e-04 loss=1.9215 ce=1.9215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7378759366692975e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007400208152830601, 'last_eos_top1': 0} +step=266600 lr=6.000000e-04 loss=2.3948 ce=2.3948 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.48368035297608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009617564268410206, 'last_eos_top1': 0} +step=266650 lr=6.000000e-04 loss=2.9642 ce=2.9642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0133480827789754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0090457359328866, 'last_eos_top1': 0} +step=266700 lr=6.000000e-04 loss=2.4345 ce=2.4345 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.712516890023835e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011657286435365677, 'last_eos_top1': 0} +step=266750 lr=6.000000e-04 loss=2.7018 ce=2.7018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.104493074119091e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010914558544754982, 'last_eos_top1': 0} +step=266800 lr=6.000000e-04 loss=2.5276 ce=2.5276 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.320259101106785e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008024407550692558, 'last_eos_top1': 0} +step=266850 lr=6.000000e-04 loss=2.3888 ce=2.3888 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.257781130145304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010571754537522793, 'last_eos_top1': 0} +step=266900 lr=6.000000e-04 loss=2.6720 ce=2.6720 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4625080439145677e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008036529645323753, 'last_eos_top1': 0} +step=266950 lr=6.000000e-04 loss=2.3763 ce=2.3763 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9546900148270652e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007605169899761677, 'last_eos_top1': 0} +step=267000 lr=6.000000e-04 loss=2.4106 ce=2.4106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.068691148655489e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008021321147680283, 'last_eos_top1': 0} +step=267050 lr=6.000000e-04 loss=2.4988 ce=2.4988 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.972275059524691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007776033133268356, 'last_eos_top1': 0} +step=267100 lr=6.000000e-04 loss=3.1662 ce=3.1662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.725722879404202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009316276758909225, 'last_eos_top1': 0} +step=267150 lr=6.000000e-04 loss=2.8460 ce=2.8460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.084072523051873e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006126516032963991, 'last_eos_top1': 0} +step=267200 lr=6.000000e-04 loss=2.0891 ce=2.0891 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6174679078394547e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005723961628973484, 'last_eos_top1': 0} +step=267250 lr=6.000000e-04 loss=2.3125 ce=2.3125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8951675883727148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008491050451993942, 'last_eos_top1': 0} +step=267300 lr=6.000000e-04 loss=2.2019 ce=2.2019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0470171875786036e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007952297106385231, 'last_eos_top1': 0} +step=267350 lr=6.000000e-04 loss=2.3589 ce=2.3589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.173555887769908e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007240789942443371, 'last_eos_top1': 0} +step=267400 lr=6.000000e-04 loss=2.7617 ce=2.7617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.672867438173853e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007255777716636658, 'last_eos_top1': 0} +step=267450 lr=6.000000e-04 loss=2.8066 ce=2.8066 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6313380658393726e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009933751076459885, 'last_eos_top1': 0} +step=267500 lr=6.000000e-04 loss=2.8505 ce=2.8505 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0788902727654204e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009073575027287006, 'last_eos_top1': 0} +step=267550 lr=6.000000e-04 loss=2.8578 ce=2.8578 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.594090453087119e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010241018608212471, 'last_eos_top1': 0} +step=267600 lr=6.000000e-04 loss=2.3592 ce=2.3592 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2066178896930069e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009177369996905327, 'last_eos_top1': 0} +step=267650 lr=6.000000e-04 loss=2.7655 ce=2.7655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.305458893824834e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012444451451301575, 'last_eos_top1': 0} +step=267700 lr=6.000000e-04 loss=2.0077 ce=2.0077 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.590374247636646e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013428064063191414, 'last_eos_top1': 0} +step=267750 lr=6.000000e-04 loss=2.4535 ce=2.4535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3065343409834895e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012471075169742107, 'last_eos_top1': 0} +step=267800 lr=6.000000e-04 loss=2.6201 ce=2.6201 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.8641162468120456e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012516258284449577, 'last_eos_top1': 0} +step=267850 lr=6.000000e-04 loss=2.7415 ce=2.7415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.077092959661968e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012690725736320019, 'last_eos_top1': 0} +step=267900 lr=6.000000e-04 loss=2.5646 ce=2.5646 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.379512574814726e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010868046432733536, 'last_eos_top1': 0} +step=267950 lr=6.000000e-04 loss=2.4090 ce=2.4090 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.495517714531161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010946475900709629, 'last_eos_top1': 0} +step=268000 lr=6.000000e-04 loss=2.3826 ce=2.3826 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4589646525564604e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009111152961850166, 'last_eos_top1': 0} +step=268050 lr=6.000000e-04 loss=2.4841 ce=2.4841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.129509296850301e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010342162102460861, 'last_eos_top1': 0} +step=268100 lr=6.000000e-04 loss=2.5462 ce=2.5462 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.340054485714063e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0100041963160038, 'last_eos_top1': 0} +step=268150 lr=6.000000e-04 loss=2.4404 ce=2.4404 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.37168590805959e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010868486948311329, 'last_eos_top1': 0} +step=268200 lr=6.000000e-04 loss=2.1538 ce=2.1538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.424078259849921e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010795933194458485, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=5 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=1] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=5 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=2] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=7 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4300000 rows=4291809 buffered=8191 part=openwebtext-t5 +step=268250 lr=6.000000e-04 loss=2.2160 ce=2.2160 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.313128430221695e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009769650176167488, 'last_eos_top1': 0} +step=268300 lr=6.000000e-04 loss=2.3174 ce=2.3174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7517193555249833e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009597726166248322, 'last_eos_top1': 0} +step=268350 lr=6.000000e-04 loss=2.6722 ce=2.6722 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5017525583971292e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01054686214774847, 'last_eos_top1': 0} +step=268400 lr=6.000000e-04 loss=2.9754 ce=2.9754 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.016620096081169e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009097069501876831, 'last_eos_top1': 0} +step=268450 lr=6.000000e-04 loss=2.4050 ce=2.4050 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.450047294492833e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006887991447001696, 'last_eos_top1': 0} +step=268500 lr=6.000000e-04 loss=1.7457 ce=1.7457 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.849022050417261e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0063669877126812935, 'last_eos_top1': 0} +step=268550 lr=6.000000e-04 loss=2.4554 ce=2.4554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.07300831284374e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006673174910247326, 'last_eos_top1': 0} +step=268600 lr=6.000000e-04 loss=2.8183 ce=2.8183 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.520665920106694e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0069860476069152355, 'last_eos_top1': 0} +step=268650 lr=6.000000e-04 loss=2.5289 ce=2.5289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.852783477981575e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008944119326770306, 'last_eos_top1': 0} +step=268700 lr=6.000000e-04 loss=2.4850 ce=2.4850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.237256194348447e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007381618022918701, 'last_eos_top1': 0} +step=268750 lr=6.000000e-04 loss=2.2015 ce=2.2015 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.399677765031811e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065951840952038765, 'last_eos_top1': 0} +step=268800 lr=6.000000e-04 loss=2.4431 ce=2.4431 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.869871190749109e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005492320284247398, 'last_eos_top1': 0} +step=268850 lr=6.000000e-04 loss=2.4589 ce=2.4589 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3442137262463802e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004075202159583569, 'last_eos_top1': 0} +step=268900 lr=6.000000e-04 loss=2.6382 ce=2.6382 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5340626709512435e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0042051710188388824, 'last_eos_top1': 0} +step=268950 lr=6.000000e-04 loss=2.2907 ce=2.2907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.92984462552704e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003420768305659294, 'last_eos_top1': 0} +step=269000 lr=6.000000e-04 loss=2.9534 ce=2.9534 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.340068234247155e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003795414697378874, 'last_eos_top1': 0} +step=269050 lr=6.000000e-04 loss=2.6992 ce=2.6992 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.913384378189221e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00392961036413908, 'last_eos_top1': 0} +step=269100 lr=6.000000e-04 loss=2.4142 ce=2.4142 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.737778454000363e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006217139773070812, 'last_eos_top1': 0} +step=269150 lr=6.000000e-04 loss=2.6353 ce=2.6353 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1576818906178232e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005837754346430302, 'last_eos_top1': 0} +step=269200 lr=6.000000e-04 loss=2.5845 ce=2.5845 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.793234883341938e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006192089058458805, 'last_eos_top1': 0} +step=269250 lr=6.000000e-04 loss=2.3751 ce=2.3751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.915818059700541e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0058769662864506245, 'last_eos_top1': 0} +step=269300 lr=6.000000e-04 loss=2.4834 ce=2.4834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.923476237512659e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006039806175976992, 'last_eos_top1': 0} +step=269350 lr=6.000000e-04 loss=2.6339 ce=2.6339 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.4558822739636526e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005903750192373991, 'last_eos_top1': 0} +step=269400 lr=6.000000e-04 loss=3.2567 ce=3.2567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.021733613830293e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0040924204513430595, 'last_eos_top1': 0} +step=269450 lr=6.000000e-04 loss=2.7482 ce=2.7482 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.463847290026024e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005740798078477383, 'last_eos_top1': 0} +step=269500 lr=6.000000e-04 loss=2.4002 ce=2.4002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.84509097179398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007107042707502842, 'last_eos_top1': 0} +step=269550 lr=6.000000e-04 loss=2.8059 ce=2.8059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.358759072900284e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007285299710929394, 'last_eos_top1': 0} +step=269600 lr=6.000000e-04 loss=2.2140 ce=2.2140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.263995909539517e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009289240464568138, 'last_eos_top1': 0} +step=269650 lr=6.000000e-04 loss=2.2886 ce=2.2886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.290154942689696e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009575280360877514, 'last_eos_top1': 0} +step=269700 lr=6.000000e-04 loss=3.1233 ce=3.1233 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1637464316154364e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009330810979008675, 'last_eos_top1': 0} +step=269750 lr=6.000000e-04 loss=2.1767 ce=2.1767 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.870018635003362e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01015952043235302, 'last_eos_top1': 0} +step=269800 lr=6.000000e-04 loss=2.4370 ce=2.4370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.492992830928415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007747658062726259, 'last_eos_top1': 0} +step=269850 lr=6.000000e-04 loss=2.1655 ce=2.1655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.164812581730075e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007498146966099739, 'last_eos_top1': 0} +step=269900 lr=6.000000e-04 loss=2.2889 ce=2.2889 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1160111171193421e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009656031616032124, 'last_eos_top1': 0} +step=269950 lr=6.000000e-04 loss=2.2944 ce=2.2944 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8490973161533475e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00916279200464487, 'last_eos_top1': 0} +step=270000 lr=6.000000e-04 loss=2.6121 ce=2.6121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.354946319537703e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01002509519457817, 'last_eos_top1': 0} +step=270050 lr=6.000000e-04 loss=2.5134 ce=2.5134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.037030395120382e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007217410020530224, 'last_eos_top1': 0} +step=270100 lr=6.000000e-04 loss=2.0718 ce=2.0718 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4825816833763383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010378338396549225, 'last_eos_top1': 0} +step=270150 lr=6.000000e-04 loss=2.3768 ce=2.3768 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.318381600867724e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012614436447620392, 'last_eos_top1': 0} +step=270200 lr=6.000000e-04 loss=2.2698 ce=2.2698 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2723965483019128e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012061797082424164, 'last_eos_top1': 0} +step=270250 lr=6.000000e-04 loss=2.6614 ce=2.6614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2469814464566298e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008977137506008148, 'last_eos_top1': 0} +step=270300 lr=6.000000e-04 loss=2.4504 ce=2.4504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.568262612447143e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010338583961129189, 'last_eos_top1': 0} +step=270350 lr=6.000000e-04 loss=2.2990 ce=2.2990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1678093869704753e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009842250496149063, 'last_eos_top1': 0} +step=270400 lr=6.000000e-04 loss=2.8480 ce=2.8480 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0738127457443625e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007883950136601925, 'last_eos_top1': 0} +step=270450 lr=6.000000e-04 loss=2.2397 ce=2.2397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1241656466154382e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009956270456314087, 'last_eos_top1': 0} +step=270500 lr=6.000000e-04 loss=3.0581 ce=3.0581 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.038265301962383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013911984860897064, 'last_eos_top1': 0} +step=270550 lr=6.000000e-04 loss=2.7732 ce=2.7732 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1992258552927524e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011622650548815727, 'last_eos_top1': 0} +step=270600 lr=6.000000e-04 loss=2.6772 ce=2.6772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6672430471517146e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010301727801561356, 'last_eos_top1': 0} +step=270650 lr=6.000000e-04 loss=2.4366 ce=2.4366 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0724346793722361e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009670315310359001, 'last_eos_top1': 0} +step=270700 lr=6.000000e-04 loss=3.4681 ce=3.4681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.61056822334649e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007666163612157106, 'last_eos_top1': 0} +step=270750 lr=6.000000e-04 loss=2.4810 ce=2.4810 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2119779057684354e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015911828726530075, 'last_eos_top1': 0} +step=270800 lr=6.000000e-04 loss=2.5771 ce=2.5771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1756703619030304e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010773276910185814, 'last_eos_top1': 0} +step=270850 lr=6.000000e-04 loss=2.4542 ce=2.4542 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0754932191048283e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007369176484644413, 'last_eos_top1': 0} +step=270900 lr=6.000000e-04 loss=2.0798 ce=2.0798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4366325861774385e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006732751615345478, 'last_eos_top1': 0} +step=270950 lr=6.000000e-04 loss=2.1999 ce=2.1999 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.114852218772285e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01049739308655262, 'last_eos_top1': 0} +step=271000 lr=6.000000e-04 loss=2.0605 ce=2.0605 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.301768058212474e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008730299770832062, 'last_eos_top1': 0} +step=271050 lr=6.000000e-04 loss=2.2536 ce=2.2536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.676262051361846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010499725118279457, 'last_eos_top1': 0} +step=271100 lr=6.000000e-04 loss=1.7590 ce=1.7590 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.324472683947533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01026462484151125, 'last_eos_top1': 0} +step=271150 lr=6.000000e-04 loss=2.4567 ce=2.4567 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.504095497599337e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009550092741847038, 'last_eos_top1': 0} +step=271200 lr=6.000000e-04 loss=2.7370 ce=2.7370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5582878404529765e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010021906346082687, 'last_eos_top1': 0} +step=271250 lr=6.000000e-04 loss=2.5498 ce=2.5498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.711592272447888e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007119513116776943, 'last_eos_top1': 0} +step=271300 lr=6.000000e-04 loss=2.8511 ce=2.8511 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2964130291948095e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00820520706474781, 'last_eos_top1': 0} +step=271350 lr=6.000000e-04 loss=2.5259 ce=2.5259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7767935787560418e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012049194425344467, 'last_eos_top1': 0} +step=271400 lr=6.000000e-04 loss=1.8840 ce=1.8840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.510358146741055e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012719842605292797, 'last_eos_top1': 0} +step=271450 lr=6.000000e-04 loss=2.4899 ce=2.4899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.610117578820791e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008246798068284988, 'last_eos_top1': 0} +step=271500 lr=6.000000e-04 loss=2.8535 ce=2.8535 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.406045137438923e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0073048765771090984, 'last_eos_top1': 0} +step=271550 lr=6.000000e-04 loss=2.2927 ce=2.2927 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.813093139295233e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00961640477180481, 'last_eos_top1': 0} +step=271600 lr=6.000000e-04 loss=2.4429 ce=2.4429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.033078079781262e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007721321191638708, 'last_eos_top1': 0} +step=271650 lr=6.000000e-04 loss=2.1302 ce=2.1302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.511290616297629e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009657420217990875, 'last_eos_top1': 0} +step=271700 lr=6.000000e-04 loss=2.1229 ce=2.1229 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0938122613879386e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010468507185578346, 'last_eos_top1': 0} +step=271750 lr=6.000000e-04 loss=2.6024 ce=2.6024 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.864730494271498e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012337048538029194, 'last_eos_top1': 0} +step=271800 lr=6.000000e-04 loss=2.5123 ce=2.5123 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3665412300033495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013169297948479652, 'last_eos_top1': 0} +step=271850 lr=6.000000e-04 loss=2.3936 ce=2.3936 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9180879462510347e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011948599480092525, 'last_eos_top1': 0} +step=271900 lr=6.000000e-04 loss=2.7618 ce=2.7618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1278261808911338e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011757656000554562, 'last_eos_top1': 0} +step=271950 lr=6.000000e-04 loss=2.3680 ce=2.3680 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.929215593729168e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00923054851591587, 'last_eos_top1': 0} +step=272000 lr=6.000000e-04 loss=2.8277 ce=2.8277 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.7085740081674885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01142309419810772, 'last_eos_top1': 0} +step=272050 lr=6.000000e-04 loss=2.7536 ce=2.7536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9377264834474772e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010125655680894852, 'last_eos_top1': 0} +step=272100 lr=6.000000e-04 loss=2.3523 ce=2.3523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.082627911993768e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0087860357016325, 'last_eos_top1': 0} +step=272150 lr=6.000000e-04 loss=2.6265 ce=2.6265 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1450485544628464e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008573771454393864, 'last_eos_top1': 0} +step=272200 lr=6.000000e-04 loss=2.7210 ce=2.7210 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.314860679092817e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009285829961299896, 'last_eos_top1': 0} +step=272250 lr=6.000000e-04 loss=2.3880 ce=2.3880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6766065527917817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009455790743231773, 'last_eos_top1': 0} +step=272300 lr=6.000000e-04 loss=2.6635 ce=2.6635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1157468179590069e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008870286867022514, 'last_eos_top1': 0} +step=272350 lr=6.000000e-04 loss=2.4835 ce=2.4835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.831888756802073e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010307036340236664, 'last_eos_top1': 0} +step=272400 lr=6.000000e-04 loss=2.7302 ce=2.7302 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4444929547607899e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0124809630215168, 'last_eos_top1': 0} +step=272450 lr=6.000000e-04 loss=2.6554 ce=2.6554 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.510098268743604e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011784246191382408, 'last_eos_top1': 0} +step=272500 lr=6.000000e-04 loss=2.4493 ce=2.4493 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5713136235717684e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011428257450461388, 'last_eos_top1': 0} +step=272550 lr=6.000000e-04 loss=2.3780 ce=2.3780 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.72070166526828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007576375268399715, 'last_eos_top1': 0} +step=272600 lr=6.000000e-04 loss=2.6726 ce=2.6726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.528966645826586e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009853953495621681, 'last_eos_top1': 0} +step=272650 lr=6.000000e-04 loss=3.0424 ce=3.0424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.176073839014862e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009013736620545387, 'last_eos_top1': 0} +step=272700 lr=6.000000e-04 loss=2.5617 ce=2.5617 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.531071787525434e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008574828505516052, 'last_eos_top1': 0} +step=272750 lr=6.000000e-04 loss=2.3403 ce=2.3403 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.853392500896007e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007992241531610489, 'last_eos_top1': 0} +step=272800 lr=6.000000e-04 loss=2.4982 ce=2.4982 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2481574231060222e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009011352434754372, 'last_eos_top1': 0} +step=272850 lr=6.000000e-04 loss=2.4377 ce=2.4377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6895552107598633e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01204107329249382, 'last_eos_top1': 0} +step=272900 lr=6.000000e-04 loss=2.1454 ce=2.1454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3971148038981482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012194247916340828, 'last_eos_top1': 0} +step=272950 lr=6.000000e-04 loss=2.1972 ce=2.1972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.086138910177397e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007767769508063793, 'last_eos_top1': 0} +step=273000 lr=6.000000e-04 loss=2.7970 ce=2.7970 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.200491836556466e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011608125641942024, 'last_eos_top1': 0} +step=273050 lr=6.000000e-04 loss=2.2791 ce=2.2791 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4091229786572512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01144565548747778, 'last_eos_top1': 0} +step=273100 lr=6.000000e-04 loss=2.3178 ce=2.3178 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.749723535089288e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009818807244300842, 'last_eos_top1': 0} +step=273150 lr=6.000000e-04 loss=2.4279 ce=2.4279 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.580103556392714e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008849301375448704, 'last_eos_top1': 0} +step=273200 lr=6.000000e-04 loss=2.0818 ce=2.0818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.99387669051066e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009304795414209366, 'last_eos_top1': 0} +step=273250 lr=6.000000e-04 loss=3.2703 ce=3.2703 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.5521538747125305e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013417836278676987, 'last_eos_top1': 0} +step=273300 lr=6.000000e-04 loss=2.4286 ce=2.4286 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.615439921617508e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011549648828804493, 'last_eos_top1': 0} +step=273350 lr=6.000000e-04 loss=2.2985 ce=2.2985 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.88645462912973e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017169956117868423, 'last_eos_top1': 0} +step=273400 lr=6.000000e-04 loss=2.4104 ce=2.4104 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.414582119556144e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012390625663101673, 'last_eos_top1': 0} +step=273450 lr=6.000000e-04 loss=2.1349 ce=2.1349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.62831246497808e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012282991781830788, 'last_eos_top1': 0} +step=273500 lr=6.000000e-04 loss=2.1378 ce=2.1378 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.267484993964899e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009551232680678368, 'last_eos_top1': 0} +step=273550 lr=6.000000e-04 loss=2.8166 ce=2.8166 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4382435438164975e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0062529356218874454, 'last_eos_top1': 0} +step=273600 lr=6.000000e-04 loss=2.0301 ce=2.0301 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2298263502307236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038908387068659067, 'last_eos_top1': 0} +step=273650 lr=6.000000e-04 loss=2.4760 ce=2.4760 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4577820770209655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004711877554655075, 'last_eos_top1': 0} +step=273700 lr=6.000000e-04 loss=2.4976 ce=2.4976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.392992418113863e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0037382692098617554, 'last_eos_top1': 0} +step=273750 lr=6.000000e-04 loss=2.8736 ce=2.8736 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8071567511651665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004964816849678755, 'last_eos_top1': 0} +step=273800 lr=6.000000e-04 loss=2.4915 ce=2.4915 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3646380466525443e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00573914498090744, 'last_eos_top1': 0} +step=273850 lr=6.000000e-04 loss=2.3729 ce=2.3729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.0091171033272985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005925910547375679, 'last_eos_top1': 0} +step=273900 lr=6.000000e-04 loss=2.4196 ce=2.4196 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.994860213017091e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00635569030418992, 'last_eos_top1': 0} +step=273950 lr=6.000000e-04 loss=2.1014 ce=2.1014 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1419453585403971e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008090082556009293, 'last_eos_top1': 0} +step=274000 lr=6.000000e-04 loss=2.1594 ce=2.1594 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2277270798222162e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008977276273071766, 'last_eos_top1': 0} +step=274050 lr=6.000000e-04 loss=1.8483 ce=1.8483 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9825230992864817e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008184470236301422, 'last_eos_top1': 0} +step=274100 lr=6.000000e-04 loss=2.4222 ce=2.4222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.123257061408367e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009919929318130016, 'last_eos_top1': 0} +step=274150 lr=6.000000e-04 loss=2.8367 ce=2.8367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.060440899891546e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00780126079916954, 'last_eos_top1': 0} +step=274200 lr=6.000000e-04 loss=2.2383 ce=2.2383 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0769963409984484e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012761814519762993, 'last_eos_top1': 0} +step=274250 lr=6.000000e-04 loss=2.2369 ce=2.2369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0120967999682762e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009586002677679062, 'last_eos_top1': 0} +step=274300 lr=6.000000e-04 loss=2.2939 ce=2.2939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2107122529414482e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008894059807062149, 'last_eos_top1': 0} +step=274350 lr=6.000000e-04 loss=2.5528 ce=2.5528 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.552539500466082e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013593384996056557, 'last_eos_top1': 0} +step=274400 lr=6.000000e-04 loss=2.8389 ce=2.8389 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1212447134312242e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012808775529265404, 'last_eos_top1': 0} +step=274450 lr=6.000000e-04 loss=2.5630 ce=2.5630 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.71419251273619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010846169665455818, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4400000 rows=4391809 buffered=8191 part=openwebtext-t5 +step=274500 lr=6.000000e-04 loss=2.3560 ce=2.3560 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0741041225846857e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012698499485850334, 'last_eos_top1': 0} +step=274550 lr=6.000000e-04 loss=2.3246 ce=2.3246 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.588857230672147e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008843690156936646, 'last_eos_top1': 0} +step=274600 lr=6.000000e-04 loss=2.3741 ce=2.3741 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1165227735764347e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009509442374110222, 'last_eos_top1': 0} +step=274650 lr=6.000000e-04 loss=2.5216 ce=2.5216 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.54024993814528e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008681866340339184, 'last_eos_top1': 0} +step=274700 lr=6.000000e-04 loss=2.3959 ce=2.3959 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4806165381742176e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009896520525217056, 'last_eos_top1': 0} +step=274750 lr=6.000000e-04 loss=2.3424 ce=2.3424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.223809678980615e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012245040386915207, 'last_eos_top1': 0} +step=274800 lr=6.000000e-04 loss=2.3017 ce=2.3017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2049066754116211e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014413516968488693, 'last_eos_top1': 0} +step=274850 lr=6.000000e-04 loss=2.6458 ce=2.6458 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.375001713517122e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013595838099718094, 'last_eos_top1': 0} +step=274900 lr=6.000000e-04 loss=2.7113 ce=2.7113 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0277928595314734e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011072298511862755, 'last_eos_top1': 0} +step=274950 lr=6.000000e-04 loss=2.6022 ce=2.6022 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.026887159445323e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013295955024659634, 'last_eos_top1': 0} +step=275000 lr=6.000000e-04 loss=2.4390 ce=2.4390 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.568806490278803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011022446677088737, 'last_eos_top1': 0} +step=275050 lr=6.000000e-04 loss=2.3909 ce=2.3909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.153193079517223e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013529550284147263, 'last_eos_top1': 0} +step=275100 lr=6.000000e-04 loss=2.3895 ce=2.3895 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.820737998059485e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013753495179116726, 'last_eos_top1': 0} +step=275150 lr=6.000000e-04 loss=2.6613 ce=2.6613 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.468145198188722e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015125194564461708, 'last_eos_top1': 0} +step=275200 lr=6.000000e-04 loss=2.9189 ce=2.9189 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.922161552822217e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017118588089942932, 'last_eos_top1': 0} +step=275250 lr=6.000000e-04 loss=2.1661 ce=2.1661 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.124778105411679e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016087248921394348, 'last_eos_top1': 0} +step=275300 lr=6.000000e-04 loss=2.4322 ce=2.4322 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.914485417539254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01389259472489357, 'last_eos_top1': 0} +step=275350 lr=6.000000e-04 loss=2.8213 ce=2.8213 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2916872947243974e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01795131526887417, 'last_eos_top1': 0} +step=275400 lr=6.000000e-04 loss=2.2814 ce=2.2814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2567637643078342e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012590310536324978, 'last_eos_top1': 0} +step=275450 lr=6.000000e-04 loss=2.3614 ce=2.3614 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.2262893152364995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012224402278661728, 'last_eos_top1': 0} +step=275500 lr=6.000000e-04 loss=2.5314 ce=2.5314 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4877018656989094e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009312121197581291, 'last_eos_top1': 0} +step=275550 lr=6.000000e-04 loss=2.3086 ce=2.3086 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.662838076299522e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01109624095261097, 'last_eos_top1': 0} +step=275600 lr=6.000000e-04 loss=2.4645 ce=2.4645 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7915983335115016e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005828732158988714, 'last_eos_top1': 0} +step=275650 lr=6.000000e-04 loss=2.0979 ce=2.0979 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7102307538152672e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0032532340846955776, 'last_eos_top1': 0} +step=275700 lr=6.000000e-04 loss=2.3905 ce=2.3905 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.530792360426858e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006112717092037201, 'last_eos_top1': 0} +step=275750 lr=6.000000e-04 loss=2.3884 ce=2.3884 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.355897074972745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013300376012921333, 'last_eos_top1': 0} +step=275800 lr=6.000000e-04 loss=2.8121 ce=2.8121 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1233907571295276e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006484533194452524, 'last_eos_top1': 0} +step=275850 lr=6.000000e-04 loss=2.6406 ce=2.6406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1246991309453733e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008653612807393074, 'last_eos_top1': 0} +step=275900 lr=6.000000e-04 loss=2.3254 ce=2.3254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0377432520035654e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01153881847858429, 'last_eos_top1': 0} +step=275950 lr=6.000000e-04 loss=2.6585 ce=2.6585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.547758984088432e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012510694563388824, 'last_eos_top1': 0} +step=276000 lr=6.000000e-04 loss=2.7706 ce=2.7706 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.906202204641886e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012712064199149609, 'last_eos_top1': 0} +step=276050 lr=6.000000e-04 loss=2.6568 ce=2.6568 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.626001206517685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014295868575572968, 'last_eos_top1': 0} +step=276100 lr=6.000000e-04 loss=2.2043 ce=2.2043 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4856786947348155e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009807785972952843, 'last_eos_top1': 0} +step=276150 lr=6.000000e-04 loss=2.4555 ce=2.4555 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3912049325881526e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004431328736245632, 'last_eos_top1': 0} +step=276200 lr=6.000000e-04 loss=2.9608 ce=2.9608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.648332707001828e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007856238633394241, 'last_eos_top1': 0} +step=276250 lr=6.000000e-04 loss=2.6023 ce=2.6023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2633502592507284e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005676053464412689, 'last_eos_top1': 0} +step=276300 lr=6.000000e-04 loss=3.1252 ce=3.1252 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.492001037055161e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005752544850111008, 'last_eos_top1': 0} +step=276350 lr=6.000000e-04 loss=3.3733 ce=3.3733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.343816832872108e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004899240098893642, 'last_eos_top1': 0} +step=276400 lr=6.000000e-04 loss=1.8369 ce=1.8369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0117439160239883e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005855667404830456, 'last_eos_top1': 0} +step=276450 lr=6.000000e-04 loss=2.2001 ce=2.2001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1084671971038915e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002639340702444315, 'last_eos_top1': 0} +step=276500 lr=6.000000e-04 loss=2.3598 ce=2.3598 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3296568542718887e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003307657316327095, 'last_eos_top1': 0} +step=276550 lr=6.000000e-04 loss=2.1185 ce=2.1185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.358896022604313e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005203810520470142, 'last_eos_top1': 0} +step=276600 lr=6.000000e-04 loss=2.2743 ce=2.2743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.229716746020131e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007246766239404678, 'last_eos_top1': 0} +step=276650 lr=6.000000e-04 loss=2.3815 ce=2.3815 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0144894986297004e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005909870844334364, 'last_eos_top1': 0} +step=276700 lr=6.000000e-04 loss=1.8647 ce=1.8647 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8229075067210943e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005968349985778332, 'last_eos_top1': 0} +step=276750 lr=6.000000e-04 loss=1.9951 ce=1.9951 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.75013938639313e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005509664304554462, 'last_eos_top1': 0} +step=276800 lr=6.000000e-04 loss=2.2642 ce=2.2642 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.867846190696582e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007067530415952206, 'last_eos_top1': 0} +step=276850 lr=6.000000e-04 loss=2.2448 ce=2.2448 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2809116014977917e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00870022363960743, 'last_eos_top1': 0} +step=276900 lr=6.000000e-04 loss=2.8583 ce=2.8583 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4312310668174177e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008833535015583038, 'last_eos_top1': 0} +step=276950 lr=6.000000e-04 loss=2.4865 ce=2.4865 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0560775081103202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0087830675765872, 'last_eos_top1': 0} +step=277000 lr=6.000000e-04 loss=2.6779 ce=2.6779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.015742054732982e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009633972309529781, 'last_eos_top1': 0} +step=277050 lr=6.000000e-04 loss=2.5586 ce=2.5586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.541992767481133e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009251766838133335, 'last_eos_top1': 0} +step=277100 lr=6.000000e-04 loss=2.3039 ce=2.3039 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.056470318872016e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012113366276025772, 'last_eos_top1': 0} +step=277150 lr=6.000000e-04 loss=2.2765 ce=2.2765 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4155548342387192e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012849877588450909, 'last_eos_top1': 0} +step=277200 lr=6.000000e-04 loss=1.9370 ce=1.9370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9983726815553382e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008243868127465248, 'last_eos_top1': 0} +step=277250 lr=6.000000e-04 loss=3.0112 ce=3.0112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.561997229757253e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009328885935246944, 'last_eos_top1': 0} +step=277300 lr=6.000000e-04 loss=2.0385 ce=2.0385 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.262415238074027e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008828305639326572, 'last_eos_top1': 0} +step=277350 lr=6.000000e-04 loss=2.7830 ce=2.7830 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0679668775992468e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006138242781162262, 'last_eos_top1': 0} +step=277400 lr=6.000000e-04 loss=2.2924 ce=2.2924 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.7030357159674168e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007293604779988527, 'last_eos_top1': 0} +step=277450 lr=6.000000e-04 loss=2.3476 ce=2.3476 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.183676406275481e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0077266632579267025, 'last_eos_top1': 0} +step=277500 lr=6.000000e-04 loss=2.0471 ce=2.0471 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.090247658605222e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00610685721039772, 'last_eos_top1': 0} +step=277550 lr=6.000000e-04 loss=2.9073 ce=2.9073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.129108204622753e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0065134488977491856, 'last_eos_top1': 0} +step=277600 lr=6.000000e-04 loss=2.4110 ce=2.4110 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4518183888867497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0066481223329901695, 'last_eos_top1': 0} +step=277650 lr=6.000000e-04 loss=2.2751 ce=2.2751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.580500096082687e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008161068893969059, 'last_eos_top1': 0} +step=277700 lr=6.000000e-04 loss=2.7856 ce=2.7856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.397920060408069e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010840540751814842, 'last_eos_top1': 0} +step=277750 lr=6.000000e-04 loss=2.5358 ce=2.5358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0332697608391754e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00756049295887351, 'last_eos_top1': 0} +step=277800 lr=6.000000e-04 loss=2.3618 ce=2.3618 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.285620762966573e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011125437915325165, 'last_eos_top1': 0} +step=277850 lr=6.000000e-04 loss=2.7117 ce=2.7117 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8442802684148774e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009810790419578552, 'last_eos_top1': 0} +step=277900 lr=6.000000e-04 loss=2.3779 ce=2.3779 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7435044355806895e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014125747606158257, 'last_eos_top1': 0} +step=277950 lr=6.000000e-04 loss=2.3283 ce=2.3283 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1246958820265718e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012197958305478096, 'last_eos_top1': 0} +step=278000 lr=6.000000e-04 loss=2.5761 ce=2.5761 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.251866681035608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010784359648823738, 'last_eos_top1': 0} +step=278050 lr=6.000000e-04 loss=2.3215 ce=2.3215 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.170553883246612e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007809824775904417, 'last_eos_top1': 0} +step=278100 lr=6.000000e-04 loss=2.4088 ce=2.4088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7080239558708854e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006073073484003544, 'last_eos_top1': 0} +step=278150 lr=6.000000e-04 loss=2.6624 ce=2.6624 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.618928182433592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008710727095603943, 'last_eos_top1': 0} +step=278200 lr=6.000000e-04 loss=2.6444 ce=2.6444 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7955529983737506e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008704792708158493, 'last_eos_top1': 0} +step=278250 lr=6.000000e-04 loss=2.4600 ce=2.4600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7977130482904613e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007811613380908966, 'last_eos_top1': 0} +step=278300 lr=6.000000e-04 loss=2.5537 ce=2.5537 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2288413927308284e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005575334187597036, 'last_eos_top1': 0} +step=278350 lr=6.000000e-04 loss=2.4907 ce=2.4907 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6400401364080608e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004085768945515156, 'last_eos_top1': 0} +step=278400 lr=6.000000e-04 loss=2.3837 ce=2.3837 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.006196705740876e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003691102145239711, 'last_eos_top1': 0} +step=278450 lr=6.000000e-04 loss=2.6585 ce=2.6585 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.625596819911152e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.003483166452497244, 'last_eos_top1': 0} +step=278500 lr=6.000000e-04 loss=2.0996 ce=2.0996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5837984392419457e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.003773793578147888, 'last_eos_top1': 0} +step=278550 lr=6.000000e-04 loss=2.6851 ce=2.6851 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.658049642283004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004644844681024551, 'last_eos_top1': 0} +step=278600 lr=6.000000e-04 loss=2.3771 ce=2.3771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.152394361270126e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006456076167523861, 'last_eos_top1': 0} +step=278650 lr=6.000000e-04 loss=2.8440 ce=2.8440 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.57528722108691e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004514227155596018, 'last_eos_top1': 0} +step=278700 lr=6.000000e-04 loss=2.4979 ce=2.4979 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5257038285199087e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0071993134915828705, 'last_eos_top1': 0} +step=278750 lr=6.000000e-04 loss=2.7953 ce=2.7953 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.887302439892665e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008345095440745354, 'last_eos_top1': 0} +step=278800 lr=6.000000e-04 loss=2.4612 ce=2.4612 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0390562238171697e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007128504104912281, 'last_eos_top1': 0} +step=278850 lr=6.000000e-04 loss=2.2259 ce=2.2259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1243093215161934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009044606238603592, 'last_eos_top1': 0} +step=278900 lr=6.000000e-04 loss=2.6501 ce=2.6501 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5240322682075202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007963120006024837, 'last_eos_top1': 0} +step=278950 lr=6.000000e-04 loss=2.5222 ce=2.5222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.679355232743546e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009314047172665596, 'last_eos_top1': 0} +step=279000 lr=6.000000e-04 loss=2.0377 ce=2.0377 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5605193513911217e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008051551878452301, 'last_eos_top1': 0} +step=279050 lr=6.000000e-04 loss=1.9597 ce=1.9597 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3611474059871398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005304382182657719, 'last_eos_top1': 0} +step=279100 lr=6.000000e-04 loss=2.4670 ce=2.4670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2072468709666282e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008347703143954277, 'last_eos_top1': 0} +step=279150 lr=6.000000e-04 loss=2.3231 ce=2.3231 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.852333849063143e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016788199543952942, 'last_eos_top1': 0} +step=279200 lr=6.000000e-04 loss=2.3367 ce=2.3367 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4190280126058497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01670149527490139, 'last_eos_top1': 0} +step=279250 lr=6.000000e-04 loss=2.5060 ce=2.5060 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.932831744663417e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010794788599014282, 'last_eos_top1': 0} +step=279300 lr=6.000000e-04 loss=1.9059 ce=1.9059 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4032796570973005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013387307524681091, 'last_eos_top1': 0} +step=279350 lr=6.000000e-04 loss=2.6632 ce=2.6632 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2482287274906412e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015407046303153038, 'last_eos_top1': 0} +step=279400 lr=6.000000e-04 loss=2.3600 ce=2.3600 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7328500689473003e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0188316497951746, 'last_eos_top1': 0} +step=279450 lr=6.000000e-04 loss=2.0394 ce=2.0394 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3143968064687215e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012824158184230328, 'last_eos_top1': 0} +step=279500 lr=6.000000e-04 loss=2.7292 ce=2.7292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1081879569683224e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008954733610153198, 'last_eos_top1': 0} +step=279550 lr=6.000000e-04 loss=2.5358 ce=2.5358 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1750725207093637e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008504999801516533, 'last_eos_top1': 0} +step=279600 lr=6.000000e-04 loss=2.4639 ce=2.4639 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0670550182112493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008354180492460728, 'last_eos_top1': 0} +step=279650 lr=6.000000e-04 loss=2.7530 ce=2.7530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.111049515311606e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0057759457267820835, 'last_eos_top1': 0} +step=279700 lr=6.000000e-04 loss=2.6892 ce=2.6892 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3168299119570293e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00588114932179451, 'last_eos_top1': 0} +step=279750 lr=6.000000e-04 loss=2.1129 ce=2.1129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5382182027678937e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0049754297360777855, 'last_eos_top1': 0} +step=279800 lr=6.000000e-04 loss=2.7140 ce=2.7140 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.064733052975498e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.002423985628411174, 'last_eos_top1': 0} +step=279850 lr=6.000000e-04 loss=2.9413 ce=2.9413 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0194045646348968e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009125247597694397, 'last_eos_top1': 0} +step=279900 lr=6.000000e-04 loss=2.3318 ce=2.3318 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.623588994145393e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008791688829660416, 'last_eos_top1': 0} +step=279950 lr=6.000000e-04 loss=2.7222 ce=2.7222 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5963141777319834e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0030952233355492353, 'last_eos_top1': 0} +step=280000 lr=6.000000e-04 loss=2.6418 ce=2.6418 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1573739357118029e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0044349730014801025, 'last_eos_top1': 0} +step=280050 lr=6.000000e-04 loss=2.9023 ce=2.9023 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0243213182548061e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0051374295726418495, 'last_eos_top1': 0} +step=280100 lr=6.000000e-04 loss=2.4675 ce=2.4675 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.461068551288918e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0038467422127723694, 'last_eos_top1': 0} +step=280150 lr=6.000000e-04 loss=2.4514 ce=2.4514 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4656530765932985e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007759021129459143, 'last_eos_top1': 0} +step=280200 lr=6.000000e-04 loss=3.1814 ce=3.1814 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3178026165405754e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005776707082986832, 'last_eos_top1': 0} +step=280250 lr=6.000000e-04 loss=2.6170 ce=2.6170 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4169407222652808e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007978664711117744, 'last_eos_top1': 0} +step=280300 lr=6.000000e-04 loss=2.4641 ce=2.4641 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.318590850336477e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008767209015786648, 'last_eos_top1': 0} +step=280350 lr=6.000000e-04 loss=2.7879 ce=2.7879 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4910814570612274e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010770536959171295, 'last_eos_top1': 0} +step=280400 lr=6.000000e-04 loss=2.3103 ce=2.3103 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.931753781216685e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0068862419575452805, 'last_eos_top1': 0} +step=280450 lr=6.000000e-04 loss=1.9977 ce=1.9977 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.638492100639269e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006415289826691151, 'last_eos_top1': 0} +step=280500 lr=6.000000e-04 loss=2.4850 ce=2.4850 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5282793103542645e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006113739684224129, 'last_eos_top1': 0} +step=280550 lr=6.000000e-04 loss=2.1033 ce=2.1033 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.003585101803765e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.005116027779877186, 'last_eos_top1': 0} +step=280600 lr=6.000000e-04 loss=3.0746 ce=3.0746 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.289118341053836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.004650118760764599, 'last_eos_top1': 0} +step=280650 lr=6.000000e-04 loss=3.0247 ce=3.0247 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0657317943696398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.004198147915303707, 'last_eos_top1': 0} +step=280700 lr=6.000000e-04 loss=2.8504 ce=2.8504 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1394163670483977e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005562176927924156, 'last_eos_top1': 0} +[elf_hfds direct rank=7 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=6 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=4 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4500000 rows=4491809 buffered=8191 part=openwebtext-t5 +step=280750 lr=6.000000e-04 loss=2.0956 ce=2.0956 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.08035453397315e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006384815089404583, 'last_eos_top1': 0} +step=280800 lr=6.000000e-04 loss=2.2019 ce=2.2019 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8125601854990236e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00847792811691761, 'last_eos_top1': 0} +step=280850 lr=6.000000e-04 loss=2.3968 ce=2.3968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3570903320214711e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007200081832706928, 'last_eos_top1': 0} +step=280900 lr=6.000000e-04 loss=2.6185 ce=2.6185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.639878953748848e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008238366805016994, 'last_eos_top1': 0} +step=280950 lr=6.000000e-04 loss=2.7164 ce=2.7164 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.075358891801443e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007539200596511364, 'last_eos_top1': 0} +step=281000 lr=6.000000e-04 loss=2.0081 ce=2.0081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1724365322152153e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007478970102965832, 'last_eos_top1': 0} +step=281050 lr=6.000000e-04 loss=2.3750 ce=2.3750 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5358727978309616e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.005803685635328293, 'last_eos_top1': 0} +step=281100 lr=6.000000e-04 loss=2.0697 ce=2.0697 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.186136255972087e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009121001698076725, 'last_eos_top1': 0} +step=281150 lr=6.000000e-04 loss=2.6034 ce=2.6034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.239845707663335e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01897401548922062, 'last_eos_top1': 0} +step=281200 lr=6.000000e-04 loss=2.8909 ce=2.8909 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0505451427889057e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011521783657371998, 'last_eos_top1': 0} +step=281250 lr=6.000000e-04 loss=2.6372 ce=2.6372 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.607233899558196e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010217479430139065, 'last_eos_top1': 0} +step=281300 lr=6.000000e-04 loss=2.2834 ce=2.2834 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.722833288426045e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00989342201501131, 'last_eos_top1': 0} +step=281350 lr=6.000000e-04 loss=2.3224 ce=2.3224 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.777796443202533e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010805032216012478, 'last_eos_top1': 0} +step=281400 lr=6.000000e-04 loss=2.5370 ce=2.5370 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8593982531456277e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015459391288459301, 'last_eos_top1': 0} +step=281450 lr=6.000000e-04 loss=2.3441 ce=2.3441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0273349289491307e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015580655075609684, 'last_eos_top1': 0} +step=281500 lr=6.000000e-04 loss=2.9437 ce=2.9437 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.9972303309477866e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01200658269226551, 'last_eos_top1': 0} +step=281550 lr=6.000000e-04 loss=2.6667 ce=2.6667 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4817256669630297e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011185355484485626, 'last_eos_top1': 0} +step=281600 lr=6.000000e-04 loss=2.4622 ce=2.4622 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1332671419950202e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010114118456840515, 'last_eos_top1': 0} +step=281650 lr=6.000000e-04 loss=2.5093 ce=2.5093 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.431248155218782e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010274896398186684, 'last_eos_top1': 0} +step=281700 lr=6.000000e-04 loss=2.4920 ce=2.4920 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.872737918049097e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008508468978106976, 'last_eos_top1': 0} +step=281750 lr=6.000000e-04 loss=2.3254 ce=2.3254 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5729952792753465e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01058906689286232, 'last_eos_top1': 0} +step=281800 lr=6.000000e-04 loss=2.6630 ce=2.6630 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0013519386120606e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0053623211570084095, 'last_eos_top1': 0} +step=281850 lr=6.000000e-04 loss=2.3230 ce=2.3230 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3136555935489014e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009937139227986336, 'last_eos_top1': 0} +step=281900 lr=6.000000e-04 loss=2.1831 ce=2.1831 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5507739590248093e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011084478348493576, 'last_eos_top1': 0} +step=281950 lr=6.000000e-04 loss=2.1047 ce=2.1047 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.224940756917931e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013630562461912632, 'last_eos_top1': 0} +step=282000 lr=6.000000e-04 loss=2.3494 ce=2.3494 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.297946073464118e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011110443621873856, 'last_eos_top1': 0} +step=282050 lr=6.000000e-04 loss=2.7304 ce=2.7304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.221215643454343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009397158399224281, 'last_eos_top1': 0} +step=282100 lr=6.000000e-04 loss=2.5310 ce=2.5310 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.451745321101043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009832971729338169, 'last_eos_top1': 0} +step=282150 lr=6.000000e-04 loss=2.8192 ce=2.8192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.091420123586431e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010839606635272503, 'last_eos_top1': 0} +step=282200 lr=6.000000e-04 loss=2.0102 ce=2.0102 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.078527930076234e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00867246650159359, 'last_eos_top1': 0} +step=282250 lr=6.000000e-04 loss=2.8058 ce=2.8058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0929295715177432e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006791288964450359, 'last_eos_top1': 0} +step=282300 lr=6.000000e-04 loss=2.6980 ce=2.6980 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8832737623597495e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008993983268737793, 'last_eos_top1': 0} +step=282350 lr=6.000000e-04 loss=2.4442 ce=2.4442 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.629368827911094e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014558134600520134, 'last_eos_top1': 0} +step=282400 lr=6.000000e-04 loss=2.7171 ce=2.7171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.576896678074263e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010763302445411682, 'last_eos_top1': 0} +step=282450 lr=6.000000e-04 loss=2.7040 ce=2.7040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.383918606734369e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013460294343531132, 'last_eos_top1': 0} +step=282500 lr=6.000000e-04 loss=2.6429 ce=2.6429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.061057537299348e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009746374562382698, 'last_eos_top1': 0} +step=282550 lr=6.000000e-04 loss=2.5491 ce=2.5491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.394728860759642e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011863352730870247, 'last_eos_top1': 0} +step=282600 lr=6.000000e-04 loss=2.4028 ce=2.4028 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.444850568077527e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008385847322642803, 'last_eos_top1': 0} +step=282650 lr=6.000000e-04 loss=2.3899 ce=2.3899 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4927521988283843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009933196008205414, 'last_eos_top1': 0} +step=282700 lr=6.000000e-04 loss=2.4082 ce=2.4082 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.995071892859414e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010839283466339111, 'last_eos_top1': 0} +step=282750 lr=6.000000e-04 loss=2.5803 ce=2.5803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1062542145955376e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.006786445155739784, 'last_eos_top1': 0} +step=282800 lr=6.000000e-04 loss=1.9304 ce=1.9304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.784876965568401e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006814953871071339, 'last_eos_top1': 0} +step=282850 lr=6.000000e-04 loss=2.5317 ce=2.5317 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.397697795066051e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00841674767434597, 'last_eos_top1': 0} +step=282900 lr=6.000000e-04 loss=2.7937 ce=2.7937 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.6469471777090803e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00872303918004036, 'last_eos_top1': 0} +step=282950 lr=6.000000e-04 loss=2.0887 ce=2.0887 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.764626388961915e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008846372365951538, 'last_eos_top1': 0} +step=283000 lr=6.000000e-04 loss=2.7596 ce=2.7596 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6347938071703538e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009848596528172493, 'last_eos_top1': 0} +step=283050 lr=6.000000e-04 loss=2.5288 ce=2.5288 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.731202246446628e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010659277439117432, 'last_eos_top1': 0} +step=283100 lr=6.000000e-04 loss=2.8199 ce=2.8199 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4112361895968206e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00983442086726427, 'last_eos_top1': 0} +step=283150 lr=6.000000e-04 loss=2.5949 ce=2.5949 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.081589809677098e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007900083437561989, 'last_eos_top1': 0} +step=283200 lr=6.000000e-04 loss=2.2858 ce=2.2858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.83168776857201e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009308120235800743, 'last_eos_top1': 0} +step=283250 lr=6.000000e-04 loss=2.3454 ce=2.3454 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.854858419392258e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010023843497037888, 'last_eos_top1': 0} +step=283300 lr=6.000000e-04 loss=2.4456 ce=2.4456 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5547215045662597e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009735208004713058, 'last_eos_top1': 0} +step=283350 lr=6.000000e-04 loss=2.8118 ce=2.8118 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0262525393045507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0133527722209692, 'last_eos_top1': 0} +step=283400 lr=6.000000e-04 loss=2.6017 ce=2.6017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8321905372431502e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.019495263695716858, 'last_eos_top1': 0} +step=283450 lr=6.000000e-04 loss=2.5506 ce=2.5506 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9449300452833995e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01244740467518568, 'last_eos_top1': 0} +step=283500 lr=6.000000e-04 loss=2.3084 ce=2.3084 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3868730093236081e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014170312322676182, 'last_eos_top1': 0} +step=283550 lr=6.000000e-04 loss=2.6972 ce=2.6972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0249543013051152e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017296800389885902, 'last_eos_top1': 0} +step=283600 lr=6.000000e-04 loss=2.0601 ce=2.0601 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2589065338252112e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009050555527210236, 'last_eos_top1': 0} +step=283650 lr=6.000000e-04 loss=2.1998 ce=2.1998 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.728075615887064e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008784754201769829, 'last_eos_top1': 0} +step=283700 lr=6.000000e-04 loss=1.9803 ce=1.9803 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0523452147026546e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009808648377656937, 'last_eos_top1': 0} +step=283750 lr=6.000000e-04 loss=3.3717 ce=3.3717 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.123510840756353e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011430863291025162, 'last_eos_top1': 0} +step=283800 lr=6.000000e-04 loss=2.9886 ce=2.9886 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6616317225270905e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011373394168913364, 'last_eos_top1': 0} +step=283850 lr=6.000000e-04 loss=2.2417 ce=2.2417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.990868986467831e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01505415141582489, 'last_eos_top1': 0} +step=283900 lr=6.000000e-04 loss=2.9463 ce=2.9463 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.897875699796714e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012244978919625282, 'last_eos_top1': 0} +step=283950 lr=6.000000e-04 loss=2.3745 ce=2.3745 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2312882972764783e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00833811890333891, 'last_eos_top1': 0} +step=284000 lr=6.000000e-04 loss=3.0001 ce=3.0001 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.114163731283043e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012845315970480442, 'last_eos_top1': 0} +step=284050 lr=6.000000e-04 loss=2.5061 ce=2.5061 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2798609279561788e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010497594252228737, 'last_eos_top1': 0} +step=284100 lr=6.000000e-04 loss=2.7498 ce=2.7498 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5469573554582894e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011512475088238716, 'last_eos_top1': 0} +step=284150 lr=6.000000e-04 loss=2.4959 ce=2.4959 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.799724698998034e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009070276282727718, 'last_eos_top1': 0} +step=284200 lr=6.000000e-04 loss=2.5798 ce=2.5798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.097881977329962e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009804660454392433, 'last_eos_top1': 0} +step=284250 lr=6.000000e-04 loss=2.3147 ce=2.3147 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1338544936734252e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01418350636959076, 'last_eos_top1': 0} +step=284300 lr=6.000000e-04 loss=2.8660 ce=2.8660 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0245701560052112e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013922685757279396, 'last_eos_top1': 0} +step=284350 lr=6.000000e-04 loss=2.4980 ce=2.4980 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2499165677581914e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014233989641070366, 'last_eos_top1': 0} +step=284400 lr=6.000000e-04 loss=2.4536 ce=2.4536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3851860785507597e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012162086553871632, 'last_eos_top1': 0} +step=284450 lr=6.000000e-04 loss=2.2496 ce=2.2496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2925784176331945e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010999804362654686, 'last_eos_top1': 0} +step=284500 lr=6.000000e-04 loss=2.4848 ce=2.4848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6150599549291655e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012838546186685562, 'last_eos_top1': 0} +step=284550 lr=6.000000e-04 loss=2.7833 ce=2.7833 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0486993232916575e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009691974148154259, 'last_eos_top1': 0} +step=284600 lr=6.000000e-04 loss=2.4737 ce=2.4737 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.268847987987101e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00866422988474369, 'last_eos_top1': 0} +step=284650 lr=6.000000e-04 loss=2.7917 ce=2.7917 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.526044793034089e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007465902250260115, 'last_eos_top1': 0} +step=284700 lr=6.000000e-04 loss=1.8074 ce=1.8074 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.083949185471283e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008449790999293327, 'last_eos_top1': 0} +step=284750 lr=6.000000e-04 loss=2.2858 ce=2.2858 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.809973562369123e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.008226832374930382, 'last_eos_top1': 0} +step=284800 lr=6.000000e-04 loss=2.5681 ce=2.5681 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.527136403950863e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009598463773727417, 'last_eos_top1': 0} +step=284850 lr=6.000000e-04 loss=2.8325 ce=2.8325 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.830018825596198e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011903829872608185, 'last_eos_top1': 0} +step=284900 lr=6.000000e-04 loss=2.3472 ce=2.3472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0659074177965522e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01222228817641735, 'last_eos_top1': 0} +step=284950 lr=6.000000e-04 loss=2.5134 ce=2.5134 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.993738912162371e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012305392883718014, 'last_eos_top1': 0} +step=285000 lr=6.000000e-04 loss=2.4348 ce=2.4348 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.093905085435836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01276983693242073, 'last_eos_top1': 0} +step=285050 lr=6.000000e-04 loss=2.5298 ce=2.5298 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0333646059734747e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013139661401510239, 'last_eos_top1': 0} +step=285100 lr=6.000000e-04 loss=2.9714 ce=2.9714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1366325452399906e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014585187658667564, 'last_eos_top1': 0} +step=285150 lr=6.000000e-04 loss=2.4426 ce=2.4426 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.08884920438868e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013700832612812519, 'last_eos_top1': 0} +step=285200 lr=6.000000e-04 loss=2.1782 ce=2.1782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.096105375443585e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010530374944210052, 'last_eos_top1': 0} +step=285250 lr=6.000000e-04 loss=2.7058 ce=2.7058 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.715100006549619e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011175567284226418, 'last_eos_top1': 0} +step=285300 lr=6.000000e-04 loss=2.6824 ce=2.6824 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2859038179158233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01019226759672165, 'last_eos_top1': 0} +step=285350 lr=6.000000e-04 loss=2.4295 ce=2.4295 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.350471150857629e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01065663993358612, 'last_eos_top1': 0} +step=285400 lr=6.000000e-04 loss=2.4075 ce=2.4075 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.578623131441418e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01108410581946373, 'last_eos_top1': 0} +step=285450 lr=6.000000e-04 loss=2.6027 ce=2.6027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.698230885784142e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010001271963119507, 'last_eos_top1': 0} +step=285500 lr=6.000000e-04 loss=2.5591 ce=2.5591 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.616950940951938e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010382583364844322, 'last_eos_top1': 0} +step=285550 lr=6.000000e-04 loss=2.5171 ce=2.5171 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.957740308484063e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012145427986979485, 'last_eos_top1': 0} +step=285600 lr=6.000000e-04 loss=2.8406 ce=2.8406 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3635353752761148e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013170614838600159, 'last_eos_top1': 0} +step=285650 lr=6.000000e-04 loss=2.3888 ce=2.3888 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.081366281141527e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011769779026508331, 'last_eos_top1': 0} +step=285700 lr=6.000000e-04 loss=2.8407 ce=2.8407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.227477150678169e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01246208418160677, 'last_eos_top1': 0} +step=285750 lr=6.000000e-04 loss=1.7772 ce=1.7772 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.547894862596877e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012072952464222908, 'last_eos_top1': 0} +step=285800 lr=6.000000e-04 loss=2.5316 ce=2.5316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.822436939226463e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012882696464657784, 'last_eos_top1': 0} +step=285850 lr=6.000000e-04 loss=2.9159 ce=2.9159 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.163809404009953e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014547569677233696, 'last_eos_top1': 0} +step=285900 lr=6.000000e-04 loss=2.3666 ce=2.3666 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5308931071776897e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014641628600656986, 'last_eos_top1': 0} +step=285950 lr=6.000000e-04 loss=2.3538 ce=2.3538 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1564087799342815e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01685246080160141, 'last_eos_top1': 0} +step=286000 lr=6.000000e-04 loss=2.5711 ce=2.5711 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0066807337570935e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010719462297856808, 'last_eos_top1': 0} +step=286050 lr=6.000000e-04 loss=2.1081 ce=2.1081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.340775821125135e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013022448867559433, 'last_eos_top1': 0} +step=286100 lr=6.000000e-04 loss=2.0685 ce=2.0685 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.1091940299083944e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01183281373232603, 'last_eos_top1': 0} +step=286150 lr=6.000000e-04 loss=2.5609 ce=2.5609 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.979375600873027e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015059503726661205, 'last_eos_top1': 0} +step=286200 lr=6.000000e-04 loss=2.2290 ce=2.2290 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4030561942490749e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011457029730081558, 'last_eos_top1': 0} +step=286250 lr=6.000000e-04 loss=2.4145 ce=2.4145 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0683899745345116e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011369128711521626, 'last_eos_top1': 0} +step=286300 lr=6.000000e-04 loss=2.8638 ce=2.8638 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4287523299572058e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010885724797844887, 'last_eos_top1': 0} +step=286350 lr=6.000000e-04 loss=2.5699 ce=2.5699 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2085130947525613e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014759449288249016, 'last_eos_top1': 0} +step=286400 lr=6.000000e-04 loss=2.5764 ce=2.5764 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4031487808097154e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016469484195113182, 'last_eos_top1': 0} +step=286450 lr=6.000000e-04 loss=2.6316 ce=2.6316 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4182660379447043e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016915708780288696, 'last_eos_top1': 0} +step=286500 lr=6.000000e-04 loss=2.0491 ce=2.0491 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.304615918546915e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016420969739556313, 'last_eos_top1': 0} +step=286550 lr=6.000000e-04 loss=2.1125 ce=2.1125 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.158239895943552e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016145501285791397, 'last_eos_top1': 0} +step=286600 lr=6.000000e-04 loss=2.2968 ce=2.2968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4060558896744624e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014938890933990479, 'last_eos_top1': 0} +step=286650 lr=6.000000e-04 loss=2.2321 ce=2.2321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.251574394700583e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012537037953734398, 'last_eos_top1': 0} +step=286700 lr=6.000000e-04 loss=2.5472 ce=2.5472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.253247309155995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011659867130219936, 'last_eos_top1': 0} +step=286750 lr=6.000000e-04 loss=2.4655 ce=2.4655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.835981821495807e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010598532855510712, 'last_eos_top1': 0} +step=286800 lr=6.000000e-04 loss=2.5374 ce=2.5374 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2750391761073843e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01452854834496975, 'last_eos_top1': 0} +step=286850 lr=6.000000e-04 loss=2.4709 ce=2.4709 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.57007250690367e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014464937150478363, 'last_eos_top1': 0} +step=286900 lr=6.000000e-04 loss=2.1914 ce=2.1914 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0328475986607373e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010671877302229404, 'last_eos_top1': 0} +step=286950 lr=6.000000e-04 loss=2.5821 ce=2.5821 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1578242265386507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008629072457551956, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=6 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=1] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=5 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=1 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=3] docs=4600000 rows=4591809 buffered=8191 part=openwebtext-t5 +step=287000 lr=6.000000e-04 loss=2.3838 ce=2.3838 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3479757399181835e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007880456745624542, 'last_eos_top1': 0} +step=287050 lr=6.000000e-04 loss=2.0536 ce=2.0536 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.742840322549455e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.008949464187026024, 'last_eos_top1': 0} +step=287100 lr=6.000000e-04 loss=2.3072 ce=2.3072 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.0647747634211555e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013212384656071663, 'last_eos_top1': 0} +step=287150 lr=6.000000e-04 loss=2.3852 ce=2.3852 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.877126674633473e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010264858603477478, 'last_eos_top1': 0} +step=287200 lr=6.000000e-04 loss=2.9928 ce=2.9928 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.147629963175859e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.006787221413105726, 'last_eos_top1': 0} +step=287250 lr=6.000000e-04 loss=2.8397 ce=2.8397 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.868442819220945e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007253343239426613, 'last_eos_top1': 0} +step=287300 lr=6.000000e-04 loss=2.7092 ce=2.7092 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1315507435938343e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.007900195196270943, 'last_eos_top1': 0} +step=287350 lr=6.000000e-04 loss=2.4096 ce=2.4096 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.561139682366047e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.007365798577666283, 'last_eos_top1': 0} +step=287400 lr=6.000000e-04 loss=2.6002 ce=2.6002 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.71478176425444e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009641788899898529, 'last_eos_top1': 0} +step=287450 lr=6.000000e-04 loss=2.5729 ce=2.5729 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.832803698373027e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009774364531040192, 'last_eos_top1': 0} +step=287500 lr=6.000000e-04 loss=2.5338 ce=2.5338 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.31887336971704e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.00964728556573391, 'last_eos_top1': 0} +step=287550 lr=6.000000e-04 loss=2.8787 ce=2.8787 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.774292392539792e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01272281352430582, 'last_eos_top1': 0} +step=287600 lr=6.000000e-04 loss=2.8417 ce=2.8417 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4995646779425442e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013687582686543465, 'last_eos_top1': 0} +step=287650 lr=6.000000e-04 loss=2.5115 ce=2.5115 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.977699508250225e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009969918057322502, 'last_eos_top1': 0} +step=287700 lr=6.000000e-04 loss=2.4092 ce=2.4092 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.694629286765121e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0096381064504385, 'last_eos_top1': 0} +step=287750 lr=6.000000e-04 loss=2.4645 ce=2.4645 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2533018889371306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009993461892008781, 'last_eos_top1': 0} +step=287800 lr=6.000000e-04 loss=2.6106 ce=2.6106 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.90770196140511e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01020287349820137, 'last_eos_top1': 0} +step=287850 lr=6.000000e-04 loss=2.3244 ce=2.3244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0132121133210603e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011937210336327553, 'last_eos_top1': 0} +step=287900 lr=6.000000e-04 loss=2.1094 ce=2.1094 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5975896531017497e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01287255622446537, 'last_eos_top1': 0} +step=287950 lr=6.000000e-04 loss=2.6862 ce=2.6862 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5419665689696558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014910340309143066, 'last_eos_top1': 0} +step=288000 lr=6.000000e-04 loss=2.5450 ce=2.5450 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.579743628914002e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01312059722840786, 'last_eos_top1': 0} +step=288050 lr=6.000000e-04 loss=2.5714 ce=2.5714 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0804684279719368e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013043233193457127, 'last_eos_top1': 0} +step=288100 lr=6.000000e-04 loss=2.1313 ce=2.1313 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.029009712918196e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.009577029384672642, 'last_eos_top1': 0} +step=288150 lr=6.000000e-04 loss=2.5563 ce=2.5563 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8689983335207216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010979942977428436, 'last_eos_top1': 0} +step=288200 lr=6.000000e-04 loss=2.3751 ce=2.3751 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2879477253591176e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01101306825876236, 'last_eos_top1': 0} +step=288250 lr=6.000000e-04 loss=2.6778 ce=2.6778 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9574155885493383e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01241311989724636, 'last_eos_top1': 0} +step=288300 lr=6.000000e-04 loss=2.3855 ce=2.3855 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.1411727832164615e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010877037420868874, 'last_eos_top1': 0} +step=288350 lr=6.000000e-04 loss=2.4120 ce=2.4120 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9652507035061717e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014871069230139256, 'last_eos_top1': 0} +step=288400 lr=6.000000e-04 loss=2.9136 ce=2.9136 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.917975719668902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017941422760486603, 'last_eos_top1': 0} +step=288450 lr=6.000000e-04 loss=2.1581 ce=2.1581 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.309316293278243e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01183172408491373, 'last_eos_top1': 0} +step=288500 lr=6.000000e-04 loss=2.3343 ce=2.3343 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.620534572401084e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013851335272192955, 'last_eos_top1': 0} +step=288550 lr=6.000000e-04 loss=2.4935 ce=2.4935 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.798746992193628e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01459646038711071, 'last_eos_top1': 0} +step=288600 lr=6.000000e-04 loss=2.1525 ce=2.1525 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.957377313985489e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013073774985969067, 'last_eos_top1': 0} +step=288650 lr=6.000000e-04 loss=3.3738 ce=3.3738 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.997662548092194e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014592032879590988, 'last_eos_top1': 0} +step=288700 lr=6.000000e-04 loss=2.5622 ce=2.5622 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5375246221083216e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016573768109083176, 'last_eos_top1': 0} +step=288750 lr=6.000000e-04 loss=2.1939 ce=2.1939 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.540829543955624e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02192595973610878, 'last_eos_top1': 0} +step=288800 lr=6.000000e-04 loss=2.6841 ce=2.6841 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.462477252469398e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016194254159927368, 'last_eos_top1': 0} +step=288850 lr=6.000000e-04 loss=2.6967 ce=2.6967 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.965312341984827e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014521583914756775, 'last_eos_top1': 0} +step=288900 lr=6.000000e-04 loss=2.1464 ce=2.1464 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.1083217082778e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013448629528284073, 'last_eos_top1': 0} +step=288950 lr=6.000000e-04 loss=3.0073 ce=3.0073 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.807554690458346e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012132110074162483, 'last_eos_top1': 0} +step=289000 lr=6.000000e-04 loss=2.9098 ce=2.9098 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.233701126097003e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015001856721937656, 'last_eos_top1': 0} +step=289050 lr=6.000000e-04 loss=2.2438 ce=2.2438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.453473128611222e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012252707034349442, 'last_eos_top1': 0} +step=289100 lr=6.000000e-04 loss=2.3832 ce=2.3832 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8115602870238945e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011499907821416855, 'last_eos_top1': 0} +step=289150 lr=6.000000e-04 loss=2.8332 ce=2.8332 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.335758133791387e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011844130232930183, 'last_eos_top1': 0} +step=289200 lr=6.000000e-04 loss=2.8131 ce=2.8131 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.511901427875273e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013745667412877083, 'last_eos_top1': 0} +step=289250 lr=6.000000e-04 loss=2.0411 ce=2.0411 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4837055864045396e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01800600066781044, 'last_eos_top1': 0} +step=289300 lr=6.000000e-04 loss=2.5968 ce=2.5968 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1167206796235405e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01743481680750847, 'last_eos_top1': 0} +step=289350 lr=6.000000e-04 loss=2.1337 ce=2.1337 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8165548681281507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.022102905437350273, 'last_eos_top1': 0} +step=289400 lr=6.000000e-04 loss=1.9798 ce=1.9798 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.683282789599616e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017281383275985718, 'last_eos_top1': 0} +step=289450 lr=6.000000e-04 loss=2.7040 ce=2.7040 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5925015759421512e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013437860645353794, 'last_eos_top1': 0} +step=289500 lr=6.000000e-04 loss=2.6017 ce=2.6017 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0699153790483251e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016002923250198364, 'last_eos_top1': 0} +step=289550 lr=6.000000e-04 loss=2.1381 ce=2.1381 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0348012185422704e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015835214406251907, 'last_eos_top1': 0} +step=289600 lr=6.000000e-04 loss=2.6270 ce=2.6270 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.238134225597605e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014819622039794922, 'last_eos_top1': 0} +step=289650 lr=6.000000e-04 loss=2.3465 ce=2.3465 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1209805052203592e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013262716121971607, 'last_eos_top1': 0} +step=289700 lr=6.000000e-04 loss=3.3236 ce=3.3236 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1213817742827814e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01695200428366661, 'last_eos_top1': 0} +step=289750 lr=6.000000e-04 loss=2.6958 ce=2.6958 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.4692652661760803e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012528153136372566, 'last_eos_top1': 0} +step=289800 lr=6.000000e-04 loss=1.9547 ce=1.9547 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3207609249075176e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01780206710100174, 'last_eos_top1': 0} +step=289850 lr=6.000000e-04 loss=2.6478 ce=2.6478 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4309691626694985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01883004792034626, 'last_eos_top1': 0} +step=289900 lr=6.000000e-04 loss=2.4662 ce=2.4662 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3913731891079806e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01592259481549263, 'last_eos_top1': 0} +step=289950 lr=6.000000e-04 loss=2.1174 ce=2.1174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.8389253455097787e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014772878959774971, 'last_eos_top1': 0} +step=290000 lr=6.000000e-04 loss=2.5674 ce=2.5674 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7034233678714372e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013806904666125774, 'last_eos_top1': 0} +step=290050 lr=6.000000e-04 loss=2.4429 ce=2.4429 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.747008905629627e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01567184180021286, 'last_eos_top1': 0} +step=290100 lr=6.000000e-04 loss=2.5393 ce=2.5393 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.165179679053836e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016209786757826805, 'last_eos_top1': 0} +step=290150 lr=6.000000e-04 loss=2.8132 ce=2.8132 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.18161788629368e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016569487750530243, 'last_eos_top1': 0} +step=290200 lr=6.000000e-04 loss=2.6743 ce=2.6743 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.141190659836866e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014392420649528503, 'last_eos_top1': 0} +step=290250 lr=6.000000e-04 loss=2.1811 ce=2.1811 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0772188034025021e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013137703761458397, 'last_eos_top1': 0} +step=290300 lr=6.000000e-04 loss=2.6848 ce=2.6848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.402039899607189e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.014557959511876106, 'last_eos_top1': 0} +step=290350 lr=6.000000e-04 loss=2.0771 ce=2.0771 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2422569852788001e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011218050494790077, 'last_eos_top1': 0} +step=290400 lr=6.000000e-04 loss=1.9319 ce=1.9319 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.068479468813166e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015913821756839752, 'last_eos_top1': 0} +step=290450 lr=6.000000e-04 loss=2.1049 ce=2.1049 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4643499525845982e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017584195360541344, 'last_eos_top1': 0} +step=290500 lr=6.000000e-04 loss=2.3991 ce=2.3991 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.9393770344322547e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013619087636470795, 'last_eos_top1': 0} +step=290550 lr=6.000000e-04 loss=1.8088 ce=1.8088 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5514198821620084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.01928405649960041, 'last_eos_top1': 0} +step=290600 lr=6.000000e-04 loss=2.5152 ce=2.5152 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.819278986658901e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018480785191059113, 'last_eos_top1': 0} +step=290650 lr=6.000000e-04 loss=2.6608 ce=2.6608 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.93347714736592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016688134521245956, 'last_eos_top1': 0} +step=290700 lr=6.000000e-04 loss=2.5289 ce=2.5289 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3670948646904435e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.021886810660362244, 'last_eos_top1': 0} +step=290750 lr=6.000000e-04 loss=2.6570 ce=2.6570 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.978131856769323e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02171536535024643, 'last_eos_top1': 0} +step=290800 lr=6.000000e-04 loss=2.2027 ce=2.2027 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.459724717278732e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.022303994745016098, 'last_eos_top1': 0} +step=290850 lr=6.000000e-04 loss=2.4122 ce=2.4122 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.8291460593172815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02319503203034401, 'last_eos_top1': 0} +step=290900 lr=6.000000e-04 loss=3.0253 ce=3.0253 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.420713467989117e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01704464480280876, 'last_eos_top1': 0} +step=290950 lr=6.000000e-04 loss=2.5192 ce=2.5192 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.382837217533961e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017804890871047974, 'last_eos_top1': 0} +step=291000 lr=6.000000e-04 loss=2.8225 ce=2.8225 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.6790230802143924e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013771768659353256, 'last_eos_top1': 0} +step=291050 lr=6.000000e-04 loss=2.4527 ce=2.4527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.136516311817104e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011629991233348846, 'last_eos_top1': 0} +step=291100 lr=6.000000e-04 loss=2.1349 ce=2.1349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.913731678854674e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.009852861985564232, 'last_eos_top1': 0} +step=291150 lr=6.000000e-04 loss=2.5972 ce=2.5972 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1175939107488375e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011300593614578247, 'last_eos_top1': 0} +step=291200 lr=6.000000e-04 loss=2.5517 ce=2.5517 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.1737487259088084e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.013259943574666977, 'last_eos_top1': 0} +step=291250 lr=6.000000e-04 loss=2.3241 ce=2.3241 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.617035741219297e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015025085769593716, 'last_eos_top1': 0} +step=291300 lr=6.000000e-04 loss=2.0034 ce=2.0034 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.99726512923371e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012920590117573738, 'last_eos_top1': 0} +step=291350 lr=6.000000e-04 loss=2.0788 ce=2.0788 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.672067264735233e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.010430430993437767, 'last_eos_top1': 0} +step=291400 lr=6.000000e-04 loss=2.3424 ce=2.3424 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.264604649506509e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.012433855794370174, 'last_eos_top1': 0} +step=291450 lr=6.000000e-04 loss=2.8715 ce=2.8715 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1232732504140586e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.0087749557569623, 'last_eos_top1': 0} +step=291500 lr=6.000000e-04 loss=2.2242 ce=2.2242 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.470040484098718e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011908701620995998, 'last_eos_top1': 0} +step=291550 lr=6.000000e-04 loss=2.6032 ce=2.6032 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.714768227946479e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010908477939665318, 'last_eos_top1': 0} +step=291600 lr=6.000000e-04 loss=2.5278 ce=2.5278 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.645156645390671e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017239684239029884, 'last_eos_top1': 0} +step=291650 lr=6.000000e-04 loss=3.0865 ce=3.0865 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.010938705003355e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016862275078892708, 'last_eos_top1': 0} +step=291700 lr=6.000000e-04 loss=2.2198 ce=2.2198 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.035241803037934e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018039144575595856, 'last_eos_top1': 0} +step=291750 lr=6.000000e-04 loss=2.3557 ce=2.3557 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1999400157947093e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02127866819500923, 'last_eos_top1': 0} +step=291800 lr=6.000000e-04 loss=2.4464 ce=2.4464 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2117373444198165e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.019953850656747818, 'last_eos_top1': 0} +step=291850 lr=6.000000e-04 loss=2.6672 ce=2.6672 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7169566490338184e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.018356481567025185, 'last_eos_top1': 0} +step=291900 lr=6.000000e-04 loss=2.6490 ce=2.6490 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.9107622948649805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.026440102607011795, 'last_eos_top1': 0} +step=291950 lr=6.000000e-04 loss=2.5862 ce=2.5862 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.945042514416855e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.03453167900443077, 'last_eos_top1': 0} +step=292000 lr=6.000000e-04 loss=2.6184 ce=2.6184 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1956150046898983e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.03815322369337082, 'last_eos_top1': 0} +step=292050 lr=6.000000e-04 loss=2.5420 ce=2.5420 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.761708962381817e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.026049785315990448, 'last_eos_top1': 0} +step=292100 lr=6.000000e-04 loss=2.4990 ce=2.4990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3418355340254493e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02682734653353691, 'last_eos_top1': 0} +step=292150 lr=6.000000e-04 loss=2.3774 ce=2.3774 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3226490409579128e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.020669810473918915, 'last_eos_top1': 0} +step=292200 lr=6.000000e-04 loss=3.1273 ce=3.1273 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3856770136917476e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017686769366264343, 'last_eos_top1': 0} +step=292250 lr=6.000000e-04 loss=2.6250 ce=2.6250 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.1542098188074306e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019546953961253166, 'last_eos_top1': 0} +step=292300 lr=6.000000e-04 loss=2.4863 ce=2.4863 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.991415830270853e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023898987099528313, 'last_eos_top1': 0} +step=292350 lr=6.000000e-04 loss=2.5870 ce=2.5870 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.239636266196612e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02187000773847103, 'last_eos_top1': 0} +step=292400 lr=6.000000e-04 loss=2.5231 ce=2.5231 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.880248060449958e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01692971959710121, 'last_eos_top1': 0} +step=292450 lr=6.000000e-04 loss=2.6691 ce=2.6691 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.6094064600765705e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010725436732172966, 'last_eos_top1': 0} +step=292500 lr=6.000000e-04 loss=2.6415 ce=2.6415 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.958771907899063e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.010566245764493942, 'last_eos_top1': 0} +step=292550 lr=6.000000e-04 loss=2.0044 ce=2.0044 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.251026312995236e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.011002952232956886, 'last_eos_top1': 0} +step=292600 lr=6.000000e-04 loss=2.3644 ce=2.3644 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.0389282983378507e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.016599101945757866, 'last_eos_top1': 0} +step=292650 lr=6.000000e-04 loss=2.6652 ce=2.6652 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7214562831213698e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.020764008164405823, 'last_eos_top1': 0} +step=292700 lr=6.000000e-04 loss=2.2569 ce=2.2569 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.2370144759188406e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.019542798399925232, 'last_eos_top1': 0} +step=292750 lr=6.000000e-04 loss=2.4897 ce=2.4897 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.210565622197464e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02437032200396061, 'last_eos_top1': 0} +step=292800 lr=6.000000e-04 loss=2.1733 ce=2.1733 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.555196366913151e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022215254604816437, 'last_eos_top1': 0} +step=292850 lr=6.000000e-04 loss=2.3527 ce=2.3527 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.433331923792139e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.032355889678001404, 'last_eos_top1': 0} +step=292900 lr=6.000000e-04 loss=2.4829 ce=2.4829 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.489185241458472e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.034103408455848694, 'last_eos_top1': 0} +step=292950 lr=6.000000e-04 loss=2.2445 ce=2.2445 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.056751666008495e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022971156984567642, 'last_eos_top1': 0} +step=293000 lr=6.000000e-04 loss=2.5129 ce=2.5129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.032649359141942e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02432781271636486, 'last_eos_top1': 0} +step=293050 lr=6.000000e-04 loss=1.9919 ce=1.9919 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.529172762588132e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02192866988480091, 'last_eos_top1': 0} +step=293100 lr=6.000000e-04 loss=2.0239 ce=2.0239 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.2116499723051675e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015998078510165215, 'last_eos_top1': 0} +step=293150 lr=6.000000e-04 loss=2.7435 ce=2.7435 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.786520887340885e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015109395608305931, 'last_eos_top1': 0} +step=293200 lr=6.000000e-04 loss=2.7211 ce=2.7211 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.876696716062725e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015001386404037476, 'last_eos_top1': 0} +[elf_hfds direct rank=5 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=0] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=0 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=1] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=4 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=1 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=3 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=0 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=2 worker=2] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=6 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=4 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=0 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=2 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5[elf_hfds direct rank=3 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 + +[elf_hfds direct rank=1 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=7 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +[elf_hfds direct rank=5 worker=3] docs=4700000 rows=4691809 buffered=8191 part=openwebtext-t5 +step=293250 lr=6.000000e-04 loss=2.6259 ce=2.6259 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.984458534338046e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012142910622060299, 'last_eos_top1': 0} +step=293300 lr=6.000000e-04 loss=2.2091 ce=2.2091 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.644127552135615e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01308517251163721, 'last_eos_top1': 0} +step=293350 lr=6.000000e-04 loss=2.7669 ce=2.7669 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.699634468532167e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023564103990793228, 'last_eos_top1': 0} +step=293400 lr=6.000000e-04 loss=2.6441 ce=2.6441 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.950712071964517e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019972115755081177, 'last_eos_top1': 0} +step=293450 lr=6.000000e-04 loss=2.8204 ce=2.8204 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.2404477678937837e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023721856996417046, 'last_eos_top1': 0} +step=293500 lr=6.000000e-04 loss=2.1701 ce=2.1701 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.46267017145874e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.020836886018514633, 'last_eos_top1': 0} +step=293550 lr=6.000000e-04 loss=2.1818 ce=2.1818 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.501637578068767e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022659532725811005, 'last_eos_top1': 0} +step=293600 lr=6.000000e-04 loss=2.4278 ce=2.4278 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.629861672990955e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02577054128050804, 'last_eos_top1': 0} +step=293650 lr=6.000000e-04 loss=2.3264 ce=2.3264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.820816229766933e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019535867497324944, 'last_eos_top1': 0} +step=293700 lr=6.000000e-04 loss=2.4680 ce=2.4680 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.907377155584982e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022013818845152855, 'last_eos_top1': 0} +step=293750 lr=6.000000e-04 loss=2.2188 ce=2.2188 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.315559064911213e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017237477004528046, 'last_eos_top1': 0} +step=293800 lr=6.000000e-04 loss=2.6973 ce=2.6973 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.009772859106306e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.017130844295024872, 'last_eos_top1': 0} +step=293850 lr=6.000000e-04 loss=2.8586 ce=2.8586 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.590179848193657e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.027764763683080673, 'last_eos_top1': 0} +step=293900 lr=6.000000e-04 loss=2.2264 ce=2.2264 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.118172339280136e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02802465297281742, 'last_eos_top1': 0} +step=293950 lr=6.000000e-04 loss=2.1410 ce=2.1410 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.3541995182749815e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.024668484926223755, 'last_eos_top1': 0} +step=294000 lr=6.000000e-04 loss=2.5304 ce=2.5304 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.293676389963366e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.027910195291042328, 'last_eos_top1': 0} +step=294050 lr=6.000000e-04 loss=3.2041 ce=3.2041 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.414721843204461e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.024418305605649948, 'last_eos_top1': 0} +step=294100 lr=6.000000e-04 loss=2.3067 ce=2.3067 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.738719432381913e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02049393206834793, 'last_eos_top1': 0} +step=294150 lr=6.000000e-04 loss=2.2603 ce=2.2603 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.202966010780074e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022201664745807648, 'last_eos_top1': 0} +step=294200 lr=6.000000e-04 loss=2.5740 ce=2.5740 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.1467717387131415e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02487393468618393, 'last_eos_top1': 0} +step=294250 lr=6.000000e-04 loss=1.6321 ce=1.6321 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0211823791905772e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02244723029434681, 'last_eos_top1': 0} +step=294300 lr=6.000000e-04 loss=3.0018 ce=3.0018 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.0360407031839713e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02726103737950325, 'last_eos_top1': 0} +step=294350 lr=6.000000e-04 loss=2.1472 ce=2.1472 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.70735892324592e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019413812085986137, 'last_eos_top1': 0} +step=294400 lr=6.000000e-04 loss=2.4349 ce=2.4349 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.387792614579666e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02207830548286438, 'last_eos_top1': 0} +step=294450 lr=6.000000e-04 loss=3.0460 ce=3.0460 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.31457248548395e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023008104413747787, 'last_eos_top1': 0} +step=294500 lr=6.000000e-04 loss=2.3840 ce=2.3840 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.935763617046177e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.026904329657554626, 'last_eos_top1': 0} +step=294550 lr=6.000000e-04 loss=2.4240 ce=2.4240 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1853175237774849e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02658362314105034, 'last_eos_top1': 0} +step=294600 lr=6.000000e-04 loss=2.2871 ce=2.2871 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4598597772419453e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02528316341340542, 'last_eos_top1': 0} +step=294650 lr=6.000000e-04 loss=2.5764 ce=2.5764 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.597588253498543e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.030090637505054474, 'last_eos_top1': 0} +step=294700 lr=6.000000e-04 loss=2.1856 ce=2.1856 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.228761776583269e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.027838440611958504, 'last_eos_top1': 0} +step=294750 lr=6.000000e-04 loss=2.5976 ce=2.5976 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5288156646420248e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.027146771550178528, 'last_eos_top1': 0} +step=294800 lr=6.000000e-04 loss=2.6245 ce=2.6245 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5025097784237005e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.028796028345823288, 'last_eos_top1': 0} +step=294850 lr=6.000000e-04 loss=2.1722 ce=2.1722 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.5609579097363167e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.028063910081982613, 'last_eos_top1': 0} +step=294900 lr=6.000000e-04 loss=2.4794 ce=2.4794 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0399947970872745e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.026524415239691734, 'last_eos_top1': 0} +step=294950 lr=6.000000e-04 loss=2.4292 ce=2.4292 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1561003702809103e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.026749897748231888, 'last_eos_top1': 0} +step=295000 lr=6.000000e-04 loss=2.1185 ce=2.1185 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.243423973093741e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.026834256947040558, 'last_eos_top1': 0} +step=295050 lr=6.000000e-04 loss=2.9265 ce=2.9265 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.901432930637384e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0237472802400589, 'last_eos_top1': 0} +step=295100 lr=6.000000e-04 loss=2.0848 ce=2.0848 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.5093853512080386e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017784275114536285, 'last_eos_top1': 0} +step=295150 lr=6.000000e-04 loss=2.1258 ce=2.1258 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.099658665130846e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.029824696481227875, 'last_eos_top1': 0} +step=295200 lr=6.000000e-04 loss=1.9107 ce=1.9107 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 9.81776429398451e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.029895741492509842, 'last_eos_top1': 0} +step=295250 lr=6.000000e-04 loss=2.8409 ce=2.8409 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.249253146961564e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.03704441338777542, 'last_eos_top1': 0} +step=295300 lr=6.000000e-04 loss=2.4074 ce=2.4074 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.3973760764638428e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02497594803571701, 'last_eos_top1': 0} +step=295350 lr=6.000000e-04 loss=2.5519 ce=2.5519 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5879220882197842e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.023250052705407143, 'last_eos_top1': 0} +step=295400 lr=6.000000e-04 loss=3.0219 ce=3.0219 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.927035767352208e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02334241382777691, 'last_eos_top1': 0} +step=295450 lr=6.000000e-04 loss=2.8244 ce=2.8244 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.547781256609596e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.022620754316449165, 'last_eos_top1': 0} +step=295500 lr=6.000000e-04 loss=2.5438 ce=2.5438 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.680473466578405e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015973884612321854, 'last_eos_top1': 0} +step=295550 lr=6.000000e-04 loss=2.6635 ce=2.6635 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.838000338873826e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015045037493109703, 'last_eos_top1': 0} +step=295600 lr=6.000000e-04 loss=2.2770 ce=2.2770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.38946938427398e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014134383760392666, 'last_eos_top1': 0} +step=295650 lr=6.000000e-04 loss=2.1323 ce=2.1323 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.885043381364085e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01491059921681881, 'last_eos_top1': 0} +step=295700 lr=6.000000e-04 loss=2.3996 ce=2.3996 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.85415876028128e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.012656887993216515, 'last_eos_top1': 0} +step=295750 lr=6.000000e-04 loss=2.6531 ce=2.6531 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.017171133658849e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013262576423585415, 'last_eos_top1': 0} +step=295800 lr=6.000000e-04 loss=2.8888 ce=2.8888 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.0831485016969964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015561539679765701, 'last_eos_top1': 0} +step=295850 lr=6.000000e-04 loss=2.8726 ce=2.8726 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.396680995138013e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.014867454767227173, 'last_eos_top1': 0} +step=295900 lr=6.000000e-04 loss=3.0209 ce=3.0209 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.82868119131308e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017761873081326485, 'last_eos_top1': 0} +step=295950 lr=6.000000e-04 loss=1.8921 ce=1.8921 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.5975985964760184e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013842686079442501, 'last_eos_top1': 0} +step=296000 lr=6.000000e-04 loss=2.5981 ce=2.5981 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1566170542209875e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.015090005472302437, 'last_eos_top1': 0} +step=296050 lr=6.000000e-04 loss=2.5407 ce=2.5407 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.946637207263848e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.015317931771278381, 'last_eos_top1': 0} +step=296100 lr=6.000000e-04 loss=2.6655 ce=2.6655 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.0709237509872764e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.01678210124373436, 'last_eos_top1': 0} +step=296150 lr=6.000000e-04 loss=2.1670 ce=2.1670 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.188759132579435e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019761625677347183, 'last_eos_top1': 0} +step=296200 lr=6.000000e-04 loss=2.2835 ce=2.2835 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.396829460049048e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.018719105049967766, 'last_eos_top1': 0} +step=296250 lr=6.000000e-04 loss=2.8369 ce=2.8369 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.2477670427178964e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.017211053520441055, 'last_eos_top1': 0} +step=296300 lr=6.000000e-04 loss=1.7963 ce=1.7963 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.3374136617348995e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.019074440002441406, 'last_eos_top1': 0} +step=296350 lr=6.000000e-04 loss=2.5902 ce=2.5902 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.4872159530059434e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023353874683380127, 'last_eos_top1': 0} +step=296400 lr=6.000000e-04 loss=2.9117 ce=2.9117 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.145459676394239e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.0141683891415596, 'last_eos_top1': 0} +step=296450 lr=6.000000e-04 loss=2.8081 ce=2.8081 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.047562470077537e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013661771081387997, 'last_eos_top1': 0} +step=296500 lr=6.000000e-04 loss=2.5045 ce=2.5045 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3032221431785729e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.011628111824393272, 'last_eos_top1': 0} +step=296550 lr=6.000000e-04 loss=2.1241 ce=2.1241 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.755947197234491e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016740238294005394, 'last_eos_top1': 0} +step=296600 lr=6.000000e-04 loss=2.7351 ce=2.7351 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.586925453826552e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.013189318589866161, 'last_eos_top1': 0} +step=296650 lr=6.000000e-04 loss=2.4011 ce=2.4011 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.332228400016902e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016763560473918915, 'last_eos_top1': 0} +step=296700 lr=6.000000e-04 loss=1.8705 ce=1.8705 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.268522393933381e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.016696225851774216, 'last_eos_top1': 0} +step=296750 lr=6.000000e-04 loss=2.1766 ce=2.1766 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.196258825890254e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02210696041584015, 'last_eos_top1': 0} +step=296800 lr=6.000000e-04 loss=2.2272 ce=2.2272 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.915855010447558e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.025051839649677277, 'last_eos_top1': 0} +step=296850 lr=6.000000e-04 loss=2.8880 ce=2.8880 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.184210749575868e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.027179505676031113, 'last_eos_top1': 0} +step=296900 lr=6.000000e-04 loss=1.9496 ce=1.9496 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.7139158444479108e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.022688157856464386, 'last_eos_top1': 0} +step=296950 lr=6.000000e-04 loss=2.4963 ce=2.4963 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.65850654715905e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.024261342361569405, 'last_eos_top1': 0} +step=297000 lr=6.000000e-04 loss=2.7112 ce=2.7112 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.511050742119551e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.031241044402122498, 'last_eos_top1': 0} +step=297050 lr=6.000000e-04 loss=2.5423 ce=2.5423 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4795392416999675e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02738041803240776, 'last_eos_top1': 0} +step=297100 lr=6.000000e-04 loss=2.7782 ce=2.7782 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.4745979569852352e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02436615712940693, 'last_eos_top1': 0} +step=297150 lr=6.000000e-04 loss=2.2757 ce=2.2757 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 3.7767818866996095e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.03374160826206207, 'last_eos_top1': 0} +step=297200 lr=6.000000e-04 loss=2.9180 ce=2.9180 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.3672526620212011e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.03967910259962082, 'last_eos_top1': 0} +step=297250 lr=6.000000e-04 loss=2.6530 ce=2.6530 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.129562047543004e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.03177002817392349, 'last_eos_top1': 0} +step=297300 lr=6.000000e-04 loss=3.0523 ce=3.0523 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.924573881406104e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.025647370144724846, 'last_eos_top1': 0} +step=297350 lr=6.000000e-04 loss=2.9037 ce=2.9037 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.5687601262470707e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.02337157167494297, 'last_eos_top1': 0} +step=297400 lr=6.000000e-04 loss=2.4129 ce=2.4129 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.493442702369066e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02199656143784523, 'last_eos_top1': 0} +step=297450 lr=6.000000e-04 loss=2.5874 ce=2.5874 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 5.3856574595556594e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02225649729371071, 'last_eos_top1': 0} +step=297500 lr=6.000000e-04 loss=2.4150 ce=2.4150 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.507651389460079e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02018890157341957, 'last_eos_top1': 0} +step=297550 lr=6.000000e-04 loss=2.5785 ce=2.5785 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 2.64422305917833e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.030234526842832565, 'last_eos_top1': 0} +step=297600 lr=6.000000e-04 loss=2.4391 ce=2.4391 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 1.1888832887052558e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.03568868339061737, 'last_eos_top1': 0} +step=297650 lr=6.000000e-04 loss=2.3770 ce=2.3770 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 8.912580597097985e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.031785234808921814, 'last_eos_top1': 0} +step=297700 lr=6.000000e-04 loss=2.6990 ce=2.6990 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 4.650118171412032e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.02811935544013977, 'last_eos_top1': 0} +step=297750 lr=6.000000e-04 loss=2.7518 ce=2.7518 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 6.142007350717904e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.020811181515455246, 'last_eos_top1': 0} +step=297800 lr=6.000000e-04 loss=2.4174 ce=2.4174 kl=0.0000 kl_w=0.0000 {'pos0_bos_p': 7.460623237420805e-06, 'pos0_bos_top1': 0, 'last_eos_p': 0.023307286202907562, 'last_eos_top1': 0} +[exit] 2026-06-09T23:08:24+00:00 rc=0 run=owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_llmclean_qwen36_35b_articlefull_10k_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_40k_elfopt_t5embed_fixed_sccfgdistill_20260529_223421.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_llmclean_qwen36_35b_articlefull_10k_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_40k_elfopt_t5embed_fixed_sccfgdistill_20260529_223421.log new file mode 100644 index 0000000000000000000000000000000000000000..a81b14ba61f28090706e53c008e99df6cb5d9783 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_t5_llmclean_qwen36_35b_articlefull_10k_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_40k_elfopt_t5embed_fixed_sccfgdistill_20260529_223421.log @@ -0,0 +1,641 @@ +W0529 22:34:26.172000 10379 torch/distributed/run.py:792] +W0529 22:34:26.172000 10379 torch/distributed/run.py:792] ***************************************** +W0529 22:34:26.172000 10379 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0529 22:34:26.172000 10379 torch/distributed/run.py:792] ***************************************** +[W529 22:34:28.765619377 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.793907386 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.798613987 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.803183472 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.804434836 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.805166862 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.805781774 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[W529 22:34:28.805905134 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +t-20260530063335-jhwp7-worker-0:10469:10469 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10469:10469 [0] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10469:10469 [0] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10469:10469 [0] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10469:10469 [0] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10472:10472 [3] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10472:10472 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10472:10472 [3] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10472:10472 [3] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10475:10475 [6] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10475:10475 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10475:10475 [6] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10475:10475 [6] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10475:10475 [6] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10472:10472 [3] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10474:10474 [5] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10474:10474 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10470:10470 [1] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10470:10470 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10471:10471 [2] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10471:10471 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10474:10474 [5] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10470:10470 [1] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10470:10470 [1] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10474:10474 [5] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10473:10473 [4] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10473:10473 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10476:10476 [7] NCCL INFO cudaDriverVersion 12080 +t-20260530063335-jhwp7-worker-0:10476:10476 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10471:10471 [2] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10471:10471 [2] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10473:10473 [4] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10476:10476 [7] NCCL INFO Bootstrap: Using eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10473:10473 [4] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10476:10476 [7] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260530063335-jhwp7-worker-0:10470:10470 [1] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10474:10474 [5] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10471:10471 [2] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10476:10476 [7] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10473:10473 [4] NCCL INFO Comm config Blocking set to 1 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO P2P plugin v9 IBext_v9 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.39<0> +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Using network IBext_v9 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO ncclCommInitRankConfig comm 0x986c380 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO ncclCommInitRankConfig comm 0x8cb0570 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO ncclCommInitRankConfig comm 0xa817420 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO ncclCommInitRankConfig comm 0x8a5bd80 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO ncclCommInitRankConfig comm 0xa63cfc0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO ncclCommInitRankConfig comm 0x8b1ff50 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO ncclCommInitRankConfig comm 0x89980c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO ncclCommInitRankConfig comm 0xa08bd40 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x88c13cf144134c5b - Init START +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO RAS client listening socket at ::1<28028> +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Bootstrap timings total 0.026253 (create 0.000028, send 0.000070, recv 0.007200, ring 0.017232, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Bootstrap timings total 0.020165 (create 0.000018, send 0.000077, recv 0.009736, ring 0.001046, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Bootstrap timings total 0.043639 (create 0.000023, send 0.000067, recv 0.008586, ring 0.007262, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Bootstrap timings total 0.016616 (create 0.000020, send 0.000068, recv 0.000079, ring 0.000108, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Bootstrap timings total 0.017680 (create 0.000026, send 0.000071, recv 0.000164, ring 0.001020, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Bootstrap timings total 0.023865 (create 0.000014, send 0.000045, recv 0.000086, ring 0.001052, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Bootstrap timings total 0.018387 (create 0.000020, send 0.000067, recv 0.000052, ring 0.000094, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Bootstrap timings total 0.037584 (create 0.000014, send 0.000044, recv 0.017395, ring 0.000110, delay 0.000001) +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO NVLS multicast support is available on dev 6 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO NVLS multicast support is available on dev 0 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO NVLS multicast support is available on dev 7 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO NVLS multicast support is available on dev 3 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO NVLS multicast support is available on dev 2 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO NVLS multicast support is available on dev 1 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO NVLS multicast support is available on dev 4 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO NVLS multicast support is available on dev 5 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO comm 0xa63cfc0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO comm 0x8b1ff50 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO comm 0x8a5bd80 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO comm 0x89980c0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO comm 0x986c380 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO comm 0x8cb0570 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO comm 0xa08bd40 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO comm 0xa817420 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO P2P Chunksize set to 524288 +t-20260530063335-jhwp7-worker-0:10472:10646 [3] NCCL INFO [Proxy Service] Device 3 CPU core 8 +t-20260530063335-jhwp7-worker-0:10472:10647 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 11 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0 +t-20260530063335-jhwp7-worker-0:10469:10648 [0] NCCL INFO [Proxy Service] Device 0 CPU core 8 +t-20260530063335-jhwp7-worker-0:10469:10649 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 11 +t-20260530063335-jhwp7-worker-0:10473:10650 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92 +t-20260530063335-jhwp7-worker-0:10473:10651 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94 +t-20260530063335-jhwp7-worker-0:10470:10652 [1] NCCL INFO [Proxy Service] Device 1 CPU core 8 +t-20260530063335-jhwp7-worker-0:10470:10653 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 11 +t-20260530063335-jhwp7-worker-0:10476:10654 [7] NCCL INFO [Proxy Service] Device 7 CPU core 152 +t-20260530063335-jhwp7-worker-0:10476:10655 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 154 +t-20260530063335-jhwp7-worker-0:10471:10656 [2] NCCL INFO [Proxy Service] Device 2 CPU core 8 +t-20260530063335-jhwp7-worker-0:10471:10657 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 11 +t-20260530063335-jhwp7-worker-0:10475:10658 [6] NCCL INFO [Proxy Service] Device 6 CPU core 106 +t-20260530063335-jhwp7-worker-0:10475:10659 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 108 +t-20260530063335-jhwp7-worker-0:10474:10660 [5] NCCL INFO [Proxy Service] Device 5 CPU core 176 +t-20260530063335-jhwp7-worker-0:10474:10661 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 178 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO CC Off, workFifoBytes 1048576 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO ncclCommInitRankConfig comm 0x8b1ff50 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10471:10547 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.12 (kernels 0.20, alloc 1.08, bootstrap 0.02, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO ncclCommInitRankConfig comm 0x8a5bd80 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO ncclCommInitRankConfig comm 0xa817420 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO ncclCommInitRankConfig comm 0xa08bd40 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO ncclCommInitRankConfig comm 0xa63cfc0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260530063335-jhwp7-worker-0:10472:10544 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.12 (kernels 0.19, alloc 1.08, bootstrap 0.02, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260530063335-jhwp7-worker-0:10470:10545 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.12 (kernels 0.19, alloc 1.08, bootstrap 0.02, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO ncclCommInitRankConfig comm 0x8cb0570 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10469:10542 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.13 (kernels 0.20, alloc 1.08, bootstrap 0.04, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO ncclCommInitRankConfig comm 0x986c380 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO ncclCommInitRankConfig comm 0x89980c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x88c13cf144134c5b - Init COMPLETE +t-20260530063335-jhwp7-worker-0:10474:10546 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.12 (kernels 0.19, alloc 1.08, bootstrap 0.02, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260530063335-jhwp7-worker-0:10476:10548 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.12 (kernels 0.19, alloc 1.06, bootstrap 0.03, allgathers 0.02, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260530063335-jhwp7-worker-0:10475:10543 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.12 (kernels 0.19, alloc 1.06, bootstrap 0.04, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.24, rest 0.03) +t-20260530063335-jhwp7-worker-0:10473:10549 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.12 (kernels 0.19, alloc 1.08, bootstrap 0.02, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.25, rest 0.02) +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260530063335-jhwp7-worker-0:10475:10662 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10474:10666 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10470:10665 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10471:10663 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10472:10664 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10469:10668 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10473:10667 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260530063335-jhwp7-worker-0:10476:10669 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +[rank0]:[W529 22:34:33.018033349 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank7]:[W529 22:34:33.018302273 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank4]:[W529 22:34:33.018329695 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank5]:[W529 22:34:33.018403234 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank6]:[W529 22:34:33.018429286 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank2]:[W529 22:34:33.018515997 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank3]:[W529 22:34:33.018535414 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +[rank1]:[W529 22:34:33.018660891 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator()) +Muon: 57 2D params; Nesterov-AdamW: 78 other params +Muon: 57 2D params; Nesterov-AdamW: 78 other paramsMuon: 57 2D params; Nesterov-AdamW: 78 other params + +Muon: 57 2D params; Nesterov-AdamW: 78 other params +Muon: 57 2D params; Nesterov-AdamW: 78 other params +Muon: 57 2D params; Nesterov-AdamW: 78 other params +Muon: 57 2D params; Nesterov-AdamW: 78 other params +Muon: 57 2D params; Nesterov-AdamW: 78 other params +{ + "data_mode": "cache", + "cache_path": "cache/owt_t5_llmclean_qwen36_35b_articlefull_pack1023_10k_appendeos1.pt", + "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext", + "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json", + "text_column": "text", + "pack_len": 1023, + "append_eos": 1, + "num_workers": 0, + "shuffle_buffer": 8192, + "reject_txt": "cache/online_rejected.txt", + "out_dir": "runs/owt_t5_llmclean_qwen36_35b_articlefull_10k_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_40k_elfopt_t5embed_fixed_sccfgdistill_20260529_223421", + "subset_size": 10000, + "resume": "", + "steps": 40000, + "batch_size": 16, + "grad_accum": 4, + "lr": 0.002, + "blr": 0.001, + "min_lr": 0.0, + "lr_schedule": "constant", + "warmup_steps": 9, + "warmup_epochs": 0.5, + "optimizer": "muon", + "weight_decay": 0.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "grad_clip": 1.0, + "log_every": 50, + "save_every": 1000, + "dim": 768, + "layers": 12, + "heads": 12, + "mlp_dim": 3072, + "time_tokens": 4, + "token_embed_init": "t5_shared", + "token_embed_model_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small", + "token_embed_dim": 512, + "freeze_token_embed": 1, + "c_min": 1.0, + "c_max": 1024.0, + "c_schedule": "exp", + "self_cond_cfg_distill": 1, + "self_cond_prob": 0.5, + "self_cond_cfg_min": 0.5, + "self_cond_cfg_max": 5.0, + "self_cond_cfg_formula": "elf", + "self_cond_kl_weight": 0.05, + "self_cond_kl_warmup": 1000, + "self_cond_teacher_temp": 1.0, + "seed": 1234, + "loader_batches_per_rank": 78, + "optimizer_steps_per_epoch": 19, + "steps_per_epoch": 19, + "effective_batch_size": 512 +} +[data] mode=cache rows=10000 length=1024 vocab=32100 seen=24862 dropped=2100 bos=1: eos=1: +[optim] optimizer=muon lr=2.000000e-03 blr=1.000000e-03 effective_batch=512 warmup_steps=9 lr_schedule=constant wd=0.0 loader_batches_per_rank=78 optimizer_steps_per_epoch=19 +[embed] init=t5_shared dim=512 freeze=True model_path=/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small +[self_cond_cfg] enabled=True prob=0.5 scale=[0.5,5.0] formula=elf kl_weight=0.05 kl_warmup=1000 teacher_temp=1.0 +t-20260530063335-jhwp7-worker-0:10476:10734 [7] NCCL INFO NVLS comm 0x8cb0570 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10475:10735 [6] NCCL INFO NVLS comm 0x986c380 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10470:10739 [1] NCCL INFO NVLS comm 0xa08bd40 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10472:10737 [3] NCCL INFO NVLS comm 0x8a5bd80 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10474:10741 [5] NCCL INFO NVLS comm 0xa63cfc0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10473:10740 [4] NCCL INFO NVLS comm 0x89980c0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10469:10736 [0] NCCL INFO NVLS comm 0xa817420 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260530063335-jhwp7-worker-0:10471:10738 [2] NCCL INFO NVLS comm 0x8b1ff50 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +step=50 lr=2.000000e-03 loss=6.8800 ce=6.8800 kl=0.0009 kl_w=0.0025 {'pos0_bos_p': 6.596895400434732e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.00013730282080359757, 'last_eos_top1': 0} +step=100 lr=2.000000e-03 loss=5.3745 ce=5.3745 kl=0.0021 kl_w=0.0050 {'pos0_bos_p': 0.0005090174963697791, 'pos0_bos_top1': 0, 'last_eos_p': 0.026919113472104073, 'last_eos_top1': 0} +step=150 lr=2.000000e-03 loss=5.4821 ce=5.4820 kl=0.0017 kl_w=0.0075 {'pos0_bos_p': 0.0005315028829500079, 'pos0_bos_top1': 0, 'last_eos_p': 0.9759315252304077, 'last_eos_top1': 4} +step=200 lr=2.000000e-03 loss=4.9991 ce=4.9990 kl=0.0049 kl_w=0.0100 {'pos0_bos_p': 0.00013899207988288254, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4} +step=250 lr=2.000000e-03 loss=4.4490 ce=4.4490 kl=0.0016 kl_w=0.0125 {'pos0_bos_p': 5.782797234132886e-05, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999727010726929, 'last_eos_top1': 4} +step=300 lr=2.000000e-03 loss=3.7176 ce=3.7176 kl=0.0031 kl_w=0.0150 {'pos0_bos_p': 0.0002401260717306286, 'pos0_bos_top1': 0, 'last_eos_p': 0.989743709564209, 'last_eos_top1': 4} +step=350 lr=2.000000e-03 loss=4.2866 ce=4.2866 kl=0.0026 kl_w=0.0175 {'pos0_bos_p': 0.0005093385698273778, 'pos0_bos_top1': 0, 'last_eos_p': 0.9409760236740112, 'last_eos_top1': 4} +step=400 lr=2.000000e-03 loss=4.3945 ce=4.3945 kl=0.0017 kl_w=0.0200 {'pos0_bos_p': 0.0005102687864564359, 'pos0_bos_top1': 0, 'last_eos_p': 0.993468701839447, 'last_eos_top1': 4} +step=450 lr=2.000000e-03 loss=3.9847 ce=3.9846 kl=0.0027 kl_w=0.0225 {'pos0_bos_p': 0.0004129198205191642, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997899532318115, 'last_eos_top1': 4} +step=500 lr=2.000000e-03 loss=3.1238 ce=3.1237 kl=0.0040 kl_w=0.0250 {'pos0_bos_p': 0.0006272579776123166, 'pos0_bos_top1': 0, 'last_eos_p': 0.9998787045478821, 'last_eos_top1': 4} +step=550 lr=2.000000e-03 loss=4.1624 ce=4.1622 kl=0.0044 kl_w=0.0275 {'pos0_bos_p': 0.0023175107780843973, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997197389602661, 'last_eos_top1': 4} +step=600 lr=2.000000e-03 loss=3.4158 ce=3.4156 kl=0.0051 kl_w=0.0300 {'pos0_bos_p': 0.0014011027524247766, 'pos0_bos_top1': 0, 'last_eos_p': 0.9976125955581665, 'last_eos_top1': 4} +step=650 lr=2.000000e-03 loss=3.4923 ce=3.4919 kl=0.0118 kl_w=0.0325 {'pos0_bos_p': 0.0017891593743115664, 'pos0_bos_top1': 0, 'last_eos_p': 0.996564507484436, 'last_eos_top1': 4} +step=700 lr=2.000000e-03 loss=3.5770 ce=3.5767 kl=0.0090 kl_w=0.0350 {'pos0_bos_p': 0.0014639467699453235, 'pos0_bos_top1': 0, 'last_eos_p': 0.9959460496902466, 'last_eos_top1': 4} +step=750 lr=2.000000e-03 loss=3.6438 ce=3.6433 kl=0.0127 kl_w=0.0375 {'pos0_bos_p': 0.0011687163496389985, 'pos0_bos_top1': 0, 'last_eos_p': 0.9943161606788635, 'last_eos_top1': 4} +step=800 lr=2.000000e-03 loss=3.3946 ce=3.3939 kl=0.0185 kl_w=0.0400 {'pos0_bos_p': 0.0013925351668149233, 'pos0_bos_top1': 0, 'last_eos_p': 0.9969325065612793, 'last_eos_top1': 4} +step=850 lr=2.000000e-03 loss=3.5824 ce=3.5816 kl=0.0198 kl_w=0.0425 {'pos0_bos_p': 0.0008899695239961147, 'pos0_bos_top1': 0, 'last_eos_p': 0.9949448108673096, 'last_eos_top1': 4} +step=900 lr=2.000000e-03 loss=3.3292 ce=3.3284 kl=0.0186 kl_w=0.0450 {'pos0_bos_p': 0.0010329543147236109, 'pos0_bos_top1': 0, 'last_eos_p': 0.9977520108222961, 'last_eos_top1': 4} +step=950 lr=2.000000e-03 loss=3.3596 ce=3.3589 kl=0.0144 kl_w=0.0475 {'pos0_bos_p': 0.0008696075528860092, 'pos0_bos_top1': 0, 'last_eos_p': 0.9979730844497681, 'last_eos_top1': 4} +step=1000 lr=2.000000e-03 loss=3.4931 ce=3.4920 kl=0.0204 kl_w=0.0500 {'pos0_bos_p': 0.0008600755827501416, 'pos0_bos_top1': 0, 'last_eos_p': 0.9988011121749878, 'last_eos_top1': 4} +step=1050 lr=2.000000e-03 loss=3.6445 ce=3.6429 kl=0.0302 kl_w=0.0500 {'pos0_bos_p': 0.000613038835581392, 'pos0_bos_top1': 0, 'last_eos_p': 0.9993675351142883, 'last_eos_top1': 4} +step=1100 lr=2.000000e-03 loss=2.8260 ce=2.8249 kl=0.0225 kl_w=0.0500 {'pos0_bos_p': 0.0005608389619737864, 'pos0_bos_top1': 0, 'last_eos_p': 0.99899822473526, 'last_eos_top1': 4} +step=1150 lr=2.000000e-03 loss=2.8863 ce=2.8850 kl=0.0267 kl_w=0.0500 {'pos0_bos_p': 0.0007489398121833801, 'pos0_bos_top1': 0, 'last_eos_p': 0.999648928642273, 'last_eos_top1': 4} +step=1200 lr=2.000000e-03 loss=2.7046 ce=2.7038 kl=0.0175 kl_w=0.0500 {'pos0_bos_p': 0.000989634427241981, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997385144233704, 'last_eos_top1': 4} +step=1250 lr=2.000000e-03 loss=2.5109 ce=2.5102 kl=0.0143 kl_w=0.0500 {'pos0_bos_p': 0.0007815308636054397, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997321367263794, 'last_eos_top1': 4} +step=1300 lr=2.000000e-03 loss=3.3680 ce=3.3669 kl=0.0221 kl_w=0.0500 {'pos0_bos_p': 0.0008436742937192321, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997445344924927, 'last_eos_top1': 4} +step=1350 lr=2.000000e-03 loss=2.6441 ce=2.6434 kl=0.0129 kl_w=0.0500 {'pos0_bos_p': 0.0007089714054018259, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997876882553101, 'last_eos_top1': 4} +step=1400 lr=2.000000e-03 loss=2.8511 ce=2.8499 kl=0.0241 kl_w=0.0500 {'pos0_bos_p': 0.0007087227422744036, 'pos0_bos_top1': 0, 'last_eos_p': 0.9998261332511902, 'last_eos_top1': 4} +step=1450 lr=2.000000e-03 loss=2.9831 ce=2.9813 kl=0.0358 kl_w=0.0500 {'pos0_bos_p': 0.0008312228601425886, 'pos0_bos_top1': 0, 'last_eos_p': 0.9997972846031189, 'last_eos_top1': 4} +step=1500 lr=2.000000e-03 loss=2.9367 ce=2.9352 kl=0.0297 kl_w=0.0500 {'pos0_bos_p': 0.001479702303186059, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999253749847412, 'last_eos_top1': 4} +step=1550 lr=2.000000e-03 loss=2.7876 ce=2.7862 kl=0.0275 kl_w=0.0500 {'pos0_bos_p': 0.0017092250054702163, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999005794525146, 'last_eos_top1': 4} +step=1600 lr=2.000000e-03 loss=3.2135 ce=3.2120 kl=0.0307 kl_w=0.0500 {'pos0_bos_p': 0.0012681031366810203, 'pos0_bos_top1': 0, 'last_eos_p': 0.9998806715011597, 'last_eos_top1': 4} +step=1650 lr=2.000000e-03 loss=3.0887 ce=3.0873 kl=0.0264 kl_w=0.0500 {'pos0_bos_p': 0.0018985029309988022, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999520778656006, 'last_eos_top1': 4} +step=1700 lr=2.000000e-03 loss=3.0711 ce=3.0697 kl=0.0284 kl_w=0.0500 {'pos0_bos_p': 0.0023892889730632305, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999046325683594, 'last_eos_top1': 4} +step=1750 lr=2.000000e-03 loss=3.2872 ce=3.2859 kl=0.0246 kl_w=0.0500 {'pos0_bos_p': 0.002865810878574848, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999246597290039, 'last_eos_top1': 4} +step=1800 lr=2.000000e-03 loss=2.8804 ce=2.8792 kl=0.0244 kl_w=0.0500 {'pos0_bos_p': 0.001702426467090845, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999648332595825, 'last_eos_top1': 4} +step=1850 lr=2.000000e-03 loss=2.4979 ce=2.4969 kl=0.0189 kl_w=0.0500 {'pos0_bos_p': 0.0022637667134404182, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999475479125977, 'last_eos_top1': 4} +step=1900 lr=2.000000e-03 loss=3.1189 ce=3.1181 kl=0.0163 kl_w=0.0500 {'pos0_bos_p': 0.00241815741173923, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999599456787109, 'last_eos_top1': 4} +step=1950 lr=2.000000e-03 loss=2.5868 ce=2.5853 kl=0.0315 kl_w=0.0500 {'pos0_bos_p': 0.0020238375291228294, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999655485153198, 'last_eos_top1': 4} +step=2000 lr=2.000000e-03 loss=3.0691 ce=3.0675 kl=0.0322 kl_w=0.0500 {'pos0_bos_p': 0.003649273421615362, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999648332595825, 'last_eos_top1': 4} +step=2050 lr=2.000000e-03 loss=3.3685 ce=3.3677 kl=0.0175 kl_w=0.0500 {'pos0_bos_p': 0.0021416968666017056, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999724626541138, 'last_eos_top1': 4} +step=2100 lr=2.000000e-03 loss=2.6890 ce=2.6875 kl=0.0298 kl_w=0.0500 {'pos0_bos_p': 0.0013540589716285467, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999639987945557, 'last_eos_top1': 4} +step=2150 lr=2.000000e-03 loss=2.8300 ce=2.8289 kl=0.0230 kl_w=0.0500 {'pos0_bos_p': 0.003231195965781808, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999499320983887, 'last_eos_top1': 4} +step=2200 lr=2.000000e-03 loss=2.5521 ce=2.5508 kl=0.0258 kl_w=0.0500 {'pos0_bos_p': 0.0019149603322148323, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999455213546753, 'last_eos_top1': 4} +step=2250 lr=2.000000e-03 loss=2.9002 ce=2.8981 kl=0.0411 kl_w=0.0500 {'pos0_bos_p': 0.002081729471683502, 'pos0_bos_top1': 0, 'last_eos_p': 0.999962329864502, 'last_eos_top1': 4} +step=2300 lr=2.000000e-03 loss=2.4927 ce=2.4890 kl=0.0723 kl_w=0.0500 {'pos0_bos_p': 0.0021949177607893944, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999468326568604, 'last_eos_top1': 4} +step=2350 lr=2.000000e-03 loss=2.9013 ce=2.9005 kl=0.0158 kl_w=0.0500 {'pos0_bos_p': 0.0016787289641797543, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999575614929199, 'last_eos_top1': 4} +step=2400 lr=2.000000e-03 loss=2.4015 ce=2.3991 kl=0.0480 kl_w=0.0500 {'pos0_bos_p': 0.0022362079471349716, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999542236328125, 'last_eos_top1': 4} +step=2450 lr=2.000000e-03 loss=2.8575 ce=2.8551 kl=0.0482 kl_w=0.0500 {'pos0_bos_p': 0.0033576919231563807, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999743700027466, 'last_eos_top1': 4} +step=2500 lr=2.000000e-03 loss=2.8900 ce=2.8876 kl=0.0463 kl_w=0.0500 {'pos0_bos_p': 0.0018122063484042883, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999603629112244, 'last_eos_top1': 4} +step=2550 lr=2.000000e-03 loss=2.2615 ce=2.2606 kl=0.0186 kl_w=0.0500 {'pos0_bos_p': 0.003717449028044939, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999768733978271, 'last_eos_top1': 4} +step=2600 lr=2.000000e-03 loss=2.8435 ce=2.8420 kl=0.0297 kl_w=0.0500 {'pos0_bos_p': 0.003102213377133012, 'pos0_bos_top1': 0, 'last_eos_p': 0.9999518394470215, 'last_eos_top1': 4} +step=2650 lr=2.000000e-03 loss=2.8277 ce=2.8248 kl=0.0581 kl_w=0.0500 {'pos0_bos_p': 0.0021677815821021795, 'pos0_bos_top1': 0, 'last_eos_p': 0.999962568283081, 'last_eos_top1': 4} +[exit] 2026-05-29T23:29:00+00:00 rc=0 run=owt_t5_llmclean_qwen36_35b_articlefull_10k_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_40k_elfopt_t5embed_fixed_sccfgdistill_20260529_223421 diff --git a/LTA_openwebtext_dualt/scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py b/LTA_openwebtext_dualt/scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..1680db86cc1108aaf8ad41ce4586ecca333ec08f --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Decode-sweep lab for FlowText OpenWebText checkpoints. + +The goal is to debug inference without touching training. We try several +simplex-valid update rules, generate many candidates, and rank them with +anti-collapse diagnostics instead of pure self-likelihood. + +Run from the flowtext_standard_bench repository root. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from collections import Counter +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Iterable, List, Sequence + +import torch +import torch.nn.functional as F + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from eval import build_model_from_ckpt +from flowtext_lab.bridges import smooth_onehot +from flowtext_lab.decode import model_time_for_step, sample_noise_simplex, state_for_model +from flowtext_lab.tokenization import BpeTextTokenizer + + +WORD_RE = re.compile(r"[A-Za-z]+|\d+|[^\sA-Za-z\d]") + + +@dataclass +class DecodeConfig: + label: str + rule: str + steps: int = 64 + model_t_mode: str = "flow" + eta: float = 0.5 + damping: float = 1.0 + max_gamma: float = 1.0 + endpoint_temp: float = 1.0 + state_floor: float = 1e-8 + final_from: str = "state" + noise_mix: float = 0.0 + noise_decay: str = "linear" + eos_logit_bias: float = 0.0 + + +def tokenize_for_metrics(text: str) -> list[str]: + return WORD_RE.findall(text) + + +def repeated_ngram_frac(tokens: Sequence[str], n: int) -> float: + if len(tokens) < n: + return 0.0 + grams = list(zip(*[tokens[i:] for i in range(n)])) + counts = Counter(grams) + return sum(v - 1 for v in counts.values() if v > 1) / max(len(grams), 1) + + +def text_metrics(text: str) -> dict: + toks = tokenize_for_metrics(text) + words = [t.lower() for t in toks if re.fullmatch(r"[A-Za-z]+", t)] + n_tok = max(len(toks), 1) + n_words = max(len(words), 1) + word_counts = Counter(words) + max_word_frac = word_counts.most_common(1)[0][1] / n_words if word_counts else 1.0 + distinct1 = len(set(words)) / n_words if words else 0.0 + bigrams = list(zip(words, words[1:])) + distinct2 = len(set(bigrams)) / max(len(bigrams), 1) if bigrams else 0.0 + digit_frac = sum(t.isdigit() for t in toks) / n_tok + punct_frac = sum(bool(re.fullmatch(r"[,.;:!?]+", t)) for t in toks) / n_tok + eos_count = text.count("<|endoftext|>") + bad_char_count = text.count("�") + rep3 = repeated_ngram_frac([t.lower() for t in toks], 3) + rep4 = repeated_ngram_frac([t.lower() for t in toks], 4) + # This score is deliberately simple and non-oracle. It rewards length and + # lexical variety while heavily penalizing classic collapse artifacts. + quality = ( + min(len(text) / 700.0, 1.0) + + 0.35 * distinct2 + + 0.15 * distinct1 + - 0.30 * eos_count + - 2.60 * rep3 + - 1.60 * rep4 + - 1.30 * digit_frac + - 0.65 * punct_frac + - 1.35 * max_word_frac + - 0.35 * bad_char_count + ) + return { + "quality": float(quality), + "chars": len(text), + "tokens": len(toks), + "words": len(words), + "eos_count": eos_count, + "bad_char_count": bad_char_count, + "rep3": float(rep3), + "rep4": float(rep4), + "distinct1": float(distinct1), + "distinct2": float(distinct2), + "digit_frac": float(digit_frac), + "punct_frac": float(punct_frac), + "max_word_frac": float(max_word_frac), + } + + +def decode_text(tokenizer: BpeTextTokenizer, ids: Sequence[int]) -> str: + return tokenizer.decode(ids, stop_at_eos=False, skip_special_tokens=False) + + +def encode_prompt(tokenizer: BpeTextTokenizer, prompt: str, max_len: int) -> list[int]: + # Prefix-continuation mode: lock [CLS] + prompt tokens, but do NOT lock an early [SEP]. + # tokenizers.Tokenizer.encode(..., add_special_tokens=False) avoids the default BERT wrapper. + core = list(tokenizer.tokenizer.encode(prompt, add_special_tokens=False).ids) + # GPT-2 uses <|endoftext|> as both BOS/EOS. For prompt continuation, + # do not lock an artificial BOS/EOT before the user prompt. + ids = core + return ids[:max_len] + + +@torch.no_grad() +def build_initial_state( + tokenizer: BpeTextTokenizer, + prompts: list[str], + restarts: int, + max_len: int, + target_prob: float, + eps: float, + noise_init: str, + noise_sigma: float, + dirichlet_init_concentration: float, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, list[str]]: + expanded: list[str] = [] + prompt_ids: list[list[int]] = [] + for prompt in prompts: + ids = encode_prompt(tokenizer, prompt, max_len=max_len) + for _ in range(restarts): + expanded.append(prompt) + prompt_ids.append(ids) + + batch = len(prompt_ids) + attn = torch.ones((batch, max_len), dtype=torch.bool, device=device) + probs = sample_noise_simplex( + (batch, max_len), + tokenizer.vocab_size, + device, + eps, + noise_mode=noise_init, + target_prob=target_prob, + noise_sigma=noise_sigma, + dirichlet_concentration=dirichlet_init_concentration, + ) + lock = torch.zeros((batch, max_len), dtype=torch.bool, device=device) + lock_probs = torch.zeros((batch, max_len, tokenizer.vocab_size), dtype=torch.float32, device=device) + for row, ids in enumerate(prompt_ids): + if not ids: + continue + ids_t = torch.tensor(ids, dtype=torch.long, device=device).unsqueeze(0) + sp = smooth_onehot(ids_t, tokenizer.vocab_size, target_prob, eps)[0] + probs[row, : len(ids)] = sp + lock_probs[row, : len(ids)] = sp + lock[row, : len(ids)] = True + return probs, attn, lock, lock_probs, expanded + + +def flowmap_gamma(step: int, steps: int, damping: float, max_gamma: float, eps: float) -> float: + s = step / max(steps, 1) + t_next = (step + 1) / max(steps, 1) + base_gamma = (t_next - s) / max(1.0 - s, eps) + gamma = float(damping) * base_gamma + return min(gamma, float(max_gamma)) if max_gamma > 0 else gamma + + +@torch.no_grad() +def decode_batch( + model, + init_probs: torch.Tensor, + attn: torch.Tensor, + lock: torch.Tensor, + lock_probs: torch.Tensor, + cfg: DecodeConfig, + eps: float, + eos_id: int | None = None, +) -> torch.Tensor: + probs = init_probs.float().clone() + device = probs.device + last_endpoint = probs + for step in range(cfg.steps): + t = model_time_for_step(cfg.model_t_mode, step, cfg.steps, probs.size(0), device, dtype=torch.float32) + logits = model(state_for_model(model, probs, eps), t, attn).float() + if cfg.endpoint_temp != 1.0: + logits = logits / float(cfg.endpoint_temp) + if cfg.eos_logit_bias != 0.0 and eos_id is not None and 0 <= eos_id < logits.size(-1): + logits[..., eos_id] = logits[..., eos_id] + float(cfg.eos_logit_bias) + endpoint = F.softmax(logits, dim=-1) + last_endpoint = endpoint + + if cfg.rule == "flowmap": + gamma = flowmap_gamma(step, cfg.steps, cfg.damping, cfg.max_gamma, eps) + new_probs = probs + gamma * (endpoint - probs) + elif cfg.rule == "replace": + new_probs = (1.0 - cfg.eta) * probs + cfg.eta * endpoint + elif cfg.rule == "geometric": + log_mix = (1.0 - cfg.eta) * torch.log(probs.clamp_min(eps)) + cfg.eta * torch.log(endpoint.clamp_min(eps)) + new_probs = F.softmax(log_mix, dim=-1) + elif cfg.rule == "centered_residual": + # Add a zero-sum probability residual, then project back to simplex. + residual = endpoint - probs + residual = residual - residual.mean(dim=-1, keepdim=True) + new_probs = probs + cfg.eta * residual + else: + raise ValueError(f"Unknown decode rule: {cfg.rule}") + + if cfg.noise_mix > 0: + if cfg.noise_decay == "linear": + lam = cfg.noise_mix * (1.0 - (step + 1) / max(cfg.steps, 1)) + elif cfg.noise_decay == "sqrt": + lam = cfg.noise_mix * math.sqrt(max(0.0, 1.0 - (step + 1) / max(cfg.steps, 1))) + else: + lam = cfg.noise_mix + if lam > 0: + uniform = torch.full_like(new_probs, 1.0 / new_probs.size(-1)) + new_probs = (1.0 - lam) * new_probs + lam * uniform + + new_probs = new_probs.clamp_min(max(float(cfg.state_floor), eps)) + new_probs = new_probs / new_probs.sum(dim=-1, keepdim=True).clamp_min(eps) + new_probs = torch.where(lock.unsqueeze(-1), lock_probs, new_probs) + probs = new_probs + + if cfg.final_from == "endpoint": + out = last_endpoint + out = torch.where(lock.unsqueeze(-1), lock_probs, out) + return out / out.sum(dim=-1, keepdim=True).clamp_min(eps) + if cfg.final_from == "blend": + out = 0.5 * probs + 0.5 * last_endpoint + out = torch.where(lock.unsqueeze(-1), lock_probs, out) + return out / out.sum(dim=-1, keepdim=True).clamp_min(eps) + return probs + + +@torch.no_grad() +def pseudo_likelihood_scores( + model, + tokenizer: BpeTextTokenizer, + probs: torch.Tensor, + attn: torch.Tensor, + lock: torch.Tensor, + target_prob: float, + eps: float, + repeats: int, + mask_frac: float, + rerank_t: float, +) -> torch.Tensor: + ids = probs.argmax(dim=-1) + endpoint = smooth_onehot(ids, tokenizer.vocab_size, target_prob, eps) + eligible = attn & (~lock) + scores = torch.zeros(ids.size(0), dtype=torch.float32, device=ids.device) + counts = torch.zeros_like(scores) + for _ in range(max(1, repeats)): + score_mask = (torch.rand_like(ids.float()) < mask_frac) & eligible + for row in range(ids.size(0)): + if eligible[row].any() and not score_mask[row].any(): + choices = torch.nonzero(eligible[row], as_tuple=False).flatten() + score_mask[row, choices[torch.randint(0, choices.numel(), (1,), device=ids.device)]] = True + noise = sample_noise_simplex( + (ids.size(0), ids.size(1)), + tokenizer.vocab_size, + ids.device, + eps, + noise_mode="logistic_normal", + target_prob=target_prob, + noise_sigma=-1.0, + ) + inp = torch.where(score_mask.unsqueeze(-1), noise, endpoint) + inp = torch.where(lock.unsqueeze(-1), probs, inp) + t = torch.full((ids.size(0),), float(rerank_t), dtype=torch.float32, device=ids.device) + logits = model(state_for_model(model, inp, eps), t, attn).float() + logp = F.log_softmax(logits, dim=-1).gather(-1, ids.unsqueeze(-1)).squeeze(-1) + scores += (logp * score_mask.float()).sum(dim=-1) + counts += score_mask.float().sum(dim=-1) + return scores / counts.clamp_min(1.0) + + +def default_configs(steps: int, config_set: str) -> list[DecodeConfig]: + if config_set == "focused_flowmap": + return [ + DecodeConfig("flowmap_t1p00_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0), + DecodeConfig("flowmap_t1p10_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.10), + DecodeConfig("flowmap_t1p25_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.25), + DecodeConfig("flowmap_t1p40_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40), + DecodeConfig("flowmap_t1p60_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.60), + DecodeConfig("flowmap_t1p25_d0p7", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25), + DecodeConfig("flowmap_t1p40_d0p7", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.40), + DecodeConfig("flowmap_t1p60_d0p7", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.60), + DecodeConfig("flowmap_t1p25_g0p5", "flowmap", steps=steps, damping=1.0, max_gamma=0.5, endpoint_temp=1.25), + DecodeConfig("flowmap_t1p40_g0p5", "flowmap", steps=steps, damping=1.0, max_gamma=0.5, endpoint_temp=1.40), + ] + if config_set == "best_flowmap": + return [ + DecodeConfig("flowmap_t1p25_d0p7", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25), + DecodeConfig("flowmap_t1p25_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.25), + DecodeConfig("flowmap_t1p35_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35), + DecodeConfig("flowmap_t1p40_d1p0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40), + ] + if config_set == "final_projection": + return [ + DecodeConfig("flowmap_t1p35_state", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, final_from="state"), + DecodeConfig("flowmap_t1p35_endpoint", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, final_from="endpoint"), + DecodeConfig("flowmap_t1p35_blend", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, final_from="blend"), + DecodeConfig("flowmap_t1p40_state", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40, final_from="state"), + DecodeConfig("flowmap_t1p40_endpoint", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40, final_from="endpoint"), + DecodeConfig("flowmap_t1p40_blend", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40, final_from="blend"), + DecodeConfig("flowmap_t1p25_d0p7_state", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25, final_from="state"), + DecodeConfig("flowmap_t1p25_d0p7_endpoint", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25, final_from="endpoint"), + DecodeConfig("flowmap_t1p25_d0p7_blend", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25, final_from="blend"), + ] + if config_set == "eos_sweep": + return [ + DecodeConfig("flowmap_t1p35_eos0", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, eos_logit_bias=0.0), + DecodeConfig("flowmap_t1p35_eos-1", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, eos_logit_bias=-1.0), + DecodeConfig("flowmap_t1p35_eos-2", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, eos_logit_bias=-2.0), + DecodeConfig("flowmap_t1p35_eos-3", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.35, eos_logit_bias=-3.0), + DecodeConfig("flowmap_t1p40_eos-2", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.40, eos_logit_bias=-2.0), + DecodeConfig("flowmap_t1p25_d0p7_eos-2", "flowmap", steps=steps, damping=0.7, max_gamma=1.0, endpoint_temp=1.25, eos_logit_bias=-2.0), + ] + if config_set != "broad": + raise ValueError(f"Unknown config_set: {config_set}") + return [ + DecodeConfig("flowmap64", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, final_from="state"), + DecodeConfig("flowmap_temp1p25", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=1.25), + DecodeConfig("flowmap_temp0p85", "flowmap", steps=steps, damping=1.0, max_gamma=1.0, endpoint_temp=0.85), + DecodeConfig("replace_eta0p35", "replace", steps=steps, eta=0.35), + DecodeConfig("replace_eta0p50", "replace", steps=steps, eta=0.50), + DecodeConfig("replace_eta0p65", "replace", steps=steps, eta=0.65), + DecodeConfig("replace_eta0p50_temp1p25", "replace", steps=steps, eta=0.50, endpoint_temp=1.25), + DecodeConfig("geometric_eta0p25", "geometric", steps=steps, eta=0.25), + DecodeConfig("geometric_eta0p50", "geometric", steps=steps, eta=0.50), + DecodeConfig("centered_residual_eta0p20", "centered_residual", steps=steps, eta=0.20), + DecodeConfig("replace_eta0p50_floor1e6", "replace", steps=steps, eta=0.50, state_floor=1e-6), + DecodeConfig("replace_eta0p50_leak", "replace", steps=steps, eta=0.50, noise_mix=0.03, noise_decay="sqrt"), + ] + + +def aggregate(rows: list[dict]) -> dict: + keys = ["quality", "eos_count", "rep3", "rep4", "distinct1", "distinct2", "digit_frac", "max_word_frac"] + return {f"mean_{k}": sum(float(r[k]) for r in rows) / max(len(rows), 1) for k in keys} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", required=True) + parser.add_argument("--tokenizer_path", required=True) + parser.add_argument("--max_len", type=int, default=128) + parser.add_argument("--steps", type=int, default=64) + parser.add_argument("--restarts", type=int, default=64) + parser.add_argument("--target_prob", type=float, default=0.99) + parser.add_argument("--eps", type=float, default=1e-8) + parser.add_argument("--model_t_mode", choices=["linear", "flow", "const0", "const05", "const1", "random"], default="flow") + parser.add_argument("--noise_init", choices=["uniform", "logistic_normal", "dirichlet"], default="dirichlet") + parser.add_argument("--noise_sigma", type=float, default=-1.0) + parser.add_argument("--dirichlet_init_concentration", type=float, default=1.0) + parser.add_argument("--prompts", default="|The|In the early morning|Scientists have|The company said|A young woman") + parser.add_argument("--score_repeats", type=int, default=0) + parser.add_argument("--score_mask_frac", type=float, default=0.5) + parser.add_argument("--rerank_t", type=float, default=0.5) + parser.add_argument("--pl_weight", type=float, default=0.0) + parser.add_argument("--output", default="runs/decode_lab/latest_decode_lab.jsonl") + parser.add_argument("--config_set", default="broad", choices=["broad", "focused_flowmap", "best_flowmap", "final_projection", "eos_sweep"]) + parser.add_argument("--decode_batch_size", type=int, default=0) + parser.add_argument("--topk", type=int, default=5) + parser.add_argument("--seed", type=int, default=20260428) + args = parser.parse_args() + + torch.manual_seed(args.seed) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path) + ckpt = torch.load(args.checkpoint, map_location="cpu") + model = build_model_from_ckpt(ckpt, tokenizer.vocab_size, args.max_len, device) + model.eval() + + prompts = args.prompts.split("|") + # Keep the first empty prompt: it is unconditional generation. + print(f"[info] device={device} prompts={prompts} restarts={args.restarts} steps={args.steps}") + print(f"[info] checkpoint={args.checkpoint}") + + out_path = Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) + configs = default_configs(args.steps, args.config_set) + for cfg in configs: + cfg.model_t_mode = args.model_t_mode + with out_path.open("w") as f: + for cfg in configs: + init, attn, lock, lock_probs, expanded = build_initial_state( + tokenizer=tokenizer, + prompts=prompts, + restarts=args.restarts, + max_len=args.max_len, + target_prob=args.target_prob, + eps=args.eps, + noise_init=args.noise_init, + noise_sigma=args.noise_sigma, + dirichlet_init_concentration=args.dirichlet_init_concentration, + device=device, + ) + if args.decode_batch_size > 0 and init.size(0) > args.decode_batch_size: + decoded_parts = [] + for start in range(0, init.size(0), args.decode_batch_size): + end = min(start + args.decode_batch_size, init.size(0)) + part = decode_batch( + model, + init[start:end], + attn[start:end], + lock[start:end], + lock_probs[start:end], + cfg, + args.eps, + tokenizer.eos_id, + ) + decoded_parts.append(part.detach().cpu()) + print(f"[chunk] {cfg.label} decoded {end}/{init.size(0)}", flush=True) + decoded = torch.cat(decoded_parts, dim=0) + else: + decoded = decode_batch(model, init, attn, lock, lock_probs, cfg, args.eps, tokenizer.eos_id) + ids = decoded.argmax(dim=-1).detach().cpu().tolist() + texts = [decode_text(tokenizer, row) for row in ids] + rows = [] + for i, text in enumerate(texts): + m = text_metrics(text) + m.update({"candidate": i, "prompt": expanded[i], "text": text}) + rows.append(m) + if args.score_repeats > 0: + decoded_for_score = decoded.to(device) if decoded.device != device else decoded + pl = pseudo_likelihood_scores( + model, + tokenizer, + decoded_for_score, + attn, + lock, + args.target_prob, + args.eps, + repeats=args.score_repeats, + mask_frac=args.score_mask_frac, + rerank_t=args.rerank_t, + ).detach().cpu().tolist() + for row, score in zip(rows, pl): + row["pseudo_logp"] = float(score) + row["rank_score"] = float(row["quality"] + args.pl_weight * score) + else: + for row in rows: + row["pseudo_logp"] = None + row["rank_score"] = float(row["quality"]) + + summary = {"type": "summary", "config": asdict(cfg), "agg": aggregate(rows)} + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + print("\n" + "=" * 96) + print("[config]", cfg.label, asdict(cfg)) + print("[metrics]", json.dumps(summary["agg"], ensure_ascii=False)) + for prompt in prompts: + subset = [r for r in rows if r["prompt"] == prompt] + subset.sort(key=lambda r: r["rank_score"], reverse=True) + for rank, row in enumerate(subset[: args.topk], 1): + rec = {"type": "sample", "config": asdict(cfg), "rank": rank, **row} + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + if rank <= 1: + print(f"\n--- best prompt={prompt!r} rank_score={row['rank_score']:.4f} quality={row['quality']:.4f} ---") + print(row["text"]) + + del init, attn, lock, lock_probs, decoded + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + print(f"[done] wrote {out_path}") + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/blockar_twostream_genppl_entropy_decode.py b/LTA_openwebtext_dualt/scripts/blockar_twostream_genppl_entropy_decode.py new file mode 100644 index 0000000000000000000000000000000000000000..1da8176cb9aa8d35158cc2c52d73a5660131a841 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/blockar_twostream_genppl_entropy_decode.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = Path(__file__).resolve().parent +for path in (REPO_ROOT, SCRIPTS_DIR): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from eval import build_model_from_ckpt +from flowtext_lab.decode import model_time_for_step, sample_noise_simplex, state_for_model +from flowtext_lab.genppl import filter_generated_texts, summarize_token_diversity +from flowtext_lab.tokenization import BpeTextTokenizer +from standard_genppl_entropy_latest_decode import ( + DecodeSetting, + current_anchor, + dirichlet_path_mean, + dirichlet_resample, + make_decode_time_grid, + make_started_state, + parse_final_from, + parse_temps, + penalty_window_scale, + sample_final_ids, + schedule_power_from_progress, + score_with_loaded, + sequence_frequency_penalty, + strip_common_special, +) + + +SPECIAL_RE = re.compile(r"\s+") + + +def make_decode_attention_mask( + *, + batch_size: int, + prefix_len: int, + block_len: int, + device: torch.device, +) -> torch.Tensor: + """Mask for a truncated [clean prefix, noisy current block] decode pass.""" + seq_len = int(prefix_len) + int(block_len) + pos = torch.cat( + [ + torch.arange(prefix_len, device=device), + torch.arange(prefix_len, prefix_len + block_len, device=device), + ], + dim=0, + ) + stream = torch.cat( + [ + torch.zeros(prefix_len, device=device, dtype=torch.long), + torch.ones(block_len, device=device, dtype=torch.long), + ], + dim=0, + ) + block = pos // int(block_len) + q_stream = stream[:, None] + k_stream = stream[None, :] + q_block = block[:, None] + k_block = block[None, :] + + clean_query = q_stream == 0 + clean_key = k_stream == 0 + noisy_query = q_stream == 1 + noisy_key = k_stream == 1 + current_block = prefix_len // int(block_len) + + clean_rule = clean_query & clean_key & (k_block <= q_block) + noisy_rule = noisy_query & ( + (clean_key & (k_block < current_block)) + | (noisy_key & (k_block == current_block)) + ) + keep = clean_rule | noisy_rule + eye = torch.eye(seq_len, device=device, dtype=torch.bool) + return (keep | eye).unsqueeze(0).expand(batch_size, -1, -1) + + +def make_decode_position_ids(prefix_len: int, block_len: int, device: torch.device) -> torch.Tensor: + return torch.cat( + [ + torch.arange(prefix_len, device=device, dtype=torch.long), + torch.arange(prefix_len, prefix_len + block_len, device=device, dtype=torch.long), + ], + dim=0, + ) + + +@dataclass +class BlockArDecodeConfig: + max_len: int + block_len: int + steps: int + model_t_mode: str + noise_init: str + dirichlet_concentration: float + concentration_min: float + concentration_max: float + noise_sigma: float + target_prob: float + decode_rule: str + support_power: float + semantic_power: float + anchor_mode: str + decode_freq_penalty_alpha: float + decode_freq_penalty_beta: float + decode_freq_penalty_floor: float + decode_freq_penalty_start: float + decode_freq_penalty_end: float + decode_freq_penalty_power: float + start_t: float + start_init: str + final_sample_mode: str + final_sample_temp: float + final_top_k: int + final_top_p: float + final_freq_penalty_alpha: float + final_freq_penalty_beta: float + final_freq_penalty_floor: float + eps: float + + +@torch.no_grad() +def decode_blockar_samples( + model, + tokenizer: BpeTextTokenizer, + setting: DecodeSetting, + cfg: BlockArDecodeConfig, + *, + n_samples: int, + batch_size: int, + decode_time_grid: list[float], + device: torch.device, +) -> tuple[list[list[int]], list[str]]: + if cfg.max_len % cfg.block_len != 0: + raise ValueError(f"max_len={cfg.max_len} must be divisible by block_len={cfg.block_len}") + + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + vocab_size = tokenizer.vocab_size + remaining = int(n_samples) + while remaining > 0: + bs = min(int(batch_size), remaining) + clean_ids = torch.zeros((bs, cfg.max_len), dtype=torch.long, device=device) + for start in range(0, cfg.max_len, cfg.block_len): + end = start + cfg.block_len + probs = make_started_state( + batch_size=bs, + max_len=cfg.block_len, + vocab_size=vocab_size, + device=device, + eps=cfg.eps, + start_t=cfg.start_t, + start_init=cfg.start_init, + noise_init=cfg.noise_init, + target_prob=cfg.target_prob, + noise_sigma=cfg.noise_sigma, + dirichlet_concentration=cfg.dirichlet_concentration, + concentration_min=cfg.concentration_min, + concentration_max=cfg.concentration_max, + ) + initial_probs = probs.clone() + last_endpoint = probs + attn = make_decode_attention_mask( + batch_size=bs, + prefix_len=start, + block_len=cfg.block_len, + device=device, + ) + position_ids = make_decode_position_ids(start, cfg.block_len, device) + + for step in range(cfg.steps): + progress = float(decode_time_grid[step]) + next_progress = float(decode_time_grid[step + 1]) + if cfg.model_t_mode == "flow": + t = torch.full((bs,), progress, device=device, dtype=torch.float32) + elif cfg.model_t_mode == "post": + t = torch.full((bs,), next_progress, device=device, dtype=torch.float32) + else: + t = model_time_for_step(cfg.model_t_mode, step, cfg.steps, bs, device, dtype=torch.float32) + + if start > 0: + prefix_probs = F.one_hot(clean_ids[:, :start], vocab_size).to(dtype=probs.dtype) + packed_probs = torch.cat([prefix_probs, probs], dim=1) + else: + packed_probs = probs + logits = model( + state_for_model(model, packed_probs, cfg.eps), + t, + attn, + position_ids=position_ids, + ).float()[:, -cfg.block_len :, :] + + decode_penalty_scale = penalty_window_scale( + next_progress, + cfg.decode_freq_penalty_start, + cfg.decode_freq_penalty_end, + cfg.decode_freq_penalty_power, + ) + if decode_penalty_scale > 0.0 and ( + cfg.decode_freq_penalty_alpha > 0.0 or cfg.decode_freq_penalty_beta > 0.0 + ): + logits = logits - decode_penalty_scale * sequence_frequency_penalty( + probs, + alpha=cfg.decode_freq_penalty_alpha, + beta=cfg.decode_freq_penalty_beta, + floor=cfg.decode_freq_penalty_floor, + eps=cfg.eps, + ).unsqueeze(-2) + + endpoint = F.softmax(logits / float(setting.endpoint_temp), dim=-1) + last_endpoint = endpoint + support_t = schedule_power_from_progress(next_progress, cfg.support_power) + if cfg.decode_rule == "flowmap": + gamma = min((next_progress - progress) / max(1.0 - progress, cfg.eps), 1.0) + probs = probs + gamma * (endpoint - probs) + probs = probs.clamp_min(cfg.eps) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(cfg.eps) + elif cfg.decode_rule == "dirichlet_mean": + probs = dirichlet_path_mean(endpoint, support_t, cfg.eps) + elif cfg.decode_rule == "dirichlet_resample": + mean = dirichlet_path_mean(endpoint, support_t, cfg.eps) + probs = dirichlet_resample(mean, support_t, cfg.concentration_min, cfg.concentration_max, cfg.eps) + elif cfg.decode_rule in {"dual_line_mean", "dual_line_resample"}: + semantic_t = schedule_power_from_progress(next_progress, cfg.semantic_power) + anchor = current_anchor(probs, cfg.anchor_mode, cfg.eps) + forward_endpoint = (1.0 - semantic_t) * anchor + semantic_t * endpoint + forward_endpoint = forward_endpoint.clamp_min(cfg.eps) + forward_endpoint = forward_endpoint / forward_endpoint.sum(dim=-1, keepdim=True).clamp_min(cfg.eps) + mean = dirichlet_path_mean(forward_endpoint, support_t, cfg.eps) + if cfg.decode_rule == "dual_line_mean": + probs = mean + else: + probs = dirichlet_resample(mean, support_t, cfg.concentration_min, cfg.concentration_max, cfg.eps) + else: + raise ValueError(f"Unknown decode_rule: {cfg.decode_rule}") + + if setting.final_from == "endpoint": + final_probs = last_endpoint + elif setting.final_from == "blend": + final_probs = 0.5 * probs + 0.5 * last_endpoint + else: + final_probs = probs + block_ids = sample_final_ids( + final_probs, + mode=cfg.final_sample_mode, + temperature=cfg.final_sample_temp, + top_k=cfg.final_top_k, + top_p=cfg.final_top_p, + freq_penalty_alpha=cfg.final_freq_penalty_alpha, + freq_penalty_beta=cfg.final_freq_penalty_beta, + freq_penalty_floor=cfg.final_freq_penalty_floor, + eps=cfg.eps, + ) + clean_ids[:, start:end] = block_ids + + ids = clean_ids.detach().cpu().tolist() + all_ids.extend(ids) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in ids) + remaining -= bs + print( + f"[blockar-decode] temp={setting.endpoint_temp:.2f} final={setting.final_from} " + f"rule={cfg.decode_rule} anchor={cfg.anchor_mode} steps={cfg.steps} " + f"generated {n_samples - remaining}/{n_samples}", + flush=True, + ) + return all_ids, all_texts + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", required=True) + parser.add_argument("--tokenizer_path", required=True) + parser.add_argument("--scorer", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--max_len", type=int, default=1024) + parser.add_argument("--block_len", type=int, default=128) + parser.add_argument("--n_samples", type=int, default=16) + parser.add_argument("--decode_batch", type=int, default=2) + parser.add_argument("--score_batch", type=int, default=8) + parser.add_argument("--score_max_length", type=int, default=256) + parser.add_argument("--steps", type=int, default=128) + parser.add_argument("--model_t_mode", default="flow") + parser.add_argument("--decode_time_schedule", choices=["linear", "sampled_path", "lognsr_gumbel"], default="linear") + parser.add_argument("--decode_s_min_frac", type=float, default=0.0) + parser.add_argument("--decode_s_max_frac", type=float, default=0.25) + parser.add_argument("--decode_time_gumbel_loc", type=float, default=2.2) + parser.add_argument("--decode_time_gumbel_scale", type=float, default=0.8) + parser.add_argument("--no_decode_force_final_t", action="store_true") + parser.add_argument( + "--decode_rule", + choices=["flowmap", "dirichlet_mean", "dirichlet_resample", "dual_line_mean", "dual_line_resample"], + default="dual_line_resample", + ) + parser.add_argument("--support_power", type=float, default=1.0) + parser.add_argument("--semantic_power", type=float, default=1.0) + parser.add_argument("--anchor_mode", choices=["onehot", "state", "sqrt_state"], default="state") + parser.add_argument("--decode_freq_penalty_alpha", type=float, default=0.0) + parser.add_argument("--decode_freq_penalty_beta", type=float, default=0.0) + parser.add_argument("--decode_freq_penalty_floor", type=float, default=0.0) + parser.add_argument("--decode_freq_penalty_start", type=float, default=0.0) + parser.add_argument("--decode_freq_penalty_end", type=float, default=1.0) + parser.add_argument("--decode_freq_penalty_power", type=float, default=1.0) + parser.add_argument("--start_t", type=float, default=0.0) + parser.add_argument( + "--start_init", + choices=["noise", "uniform_mean", "uniform_dirichlet", "random_anchor_mean", "random_anchor_dirichlet"], + default="noise", + ) + parser.add_argument("--noise_init", choices=["uniform", "logistic_normal", "dirichlet"], default="dirichlet") + parser.add_argument("--noise_sigma", type=float, default=-1.0) + parser.add_argument("--dirichlet_concentration", type=float, default=-1.0) + parser.add_argument("--concentration_min", type=float, default=1.0) + parser.add_argument("--concentration_max", type=float, default=1024.0) + parser.add_argument("--target_prob", type=float, default=-1.0) + parser.add_argument("--endpoint_temps", default="1.45") + parser.add_argument("--final_from", default="state") + parser.add_argument("--final_sample_mode", choices=["argmax", "sample", "topk", "topp"], default="argmax") + parser.add_argument("--final_sample_temp", type=float, default=1.0) + parser.add_argument("--final_top_k", type=int, default=64) + parser.add_argument("--final_top_p", type=float, default=0.95) + parser.add_argument("--final_freq_penalty_alpha", type=float, default=0.0) + parser.add_argument("--final_freq_penalty_beta", type=float, default=0.0) + parser.add_argument("--final_freq_penalty_floor", type=float, default=0.0) + parser.add_argument("--eps", type=float, default=1e-8) + parser.add_argument("--seed", type=int, default=20260522) + parser.add_argument("--save_samples", type=int, default=16) + args = parser.parse_args() + + torch.manual_seed(args.seed) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path) + ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) + ckpt_args = ckpt.get("args", {}) + step = ckpt.get("step") + target_prob = float(args.target_prob if args.target_prob >= 0 else ckpt_args.get("target_prob", 1.0)) + dirichlet_concentration = float( + args.dirichlet_concentration + if args.dirichlet_concentration > 0 + else ckpt_args.get("dirichlet_concentration_min", 1.0) + ) + print(f"[ckpt] {args.checkpoint} step={step}", flush=True) + print( + f"[blockar-base] n={args.n_samples} max_len={args.max_len} block_len={args.block_len} " + f"steps={args.steps} batch={args.decode_batch}", + flush=True, + ) + + decode_time_grid = make_decode_time_grid( + steps=args.steps, + start_t=args.start_t, + schedule=args.decode_time_schedule, + s_min_frac=args.decode_s_min_frac, + s_max_frac=args.decode_s_max_frac, + gumbel_loc=args.decode_time_gumbel_loc, + gumbel_scale=args.decode_time_gumbel_scale, + seed=args.seed, + force_final_t=not args.no_decode_force_final_t, + ) + model = build_model_from_ckpt(ckpt, tokenizer.vocab_size, args.max_len * 2, device).eval() + + cfg = BlockArDecodeConfig( + max_len=args.max_len, + block_len=args.block_len, + steps=args.steps, + model_t_mode=args.model_t_mode, + noise_init=args.noise_init, + dirichlet_concentration=dirichlet_concentration, + concentration_min=args.concentration_min, + concentration_max=args.concentration_max, + noise_sigma=args.noise_sigma, + target_prob=target_prob, + decode_rule=args.decode_rule, + support_power=args.support_power, + semantic_power=args.semantic_power, + anchor_mode=args.anchor_mode, + decode_freq_penalty_alpha=args.decode_freq_penalty_alpha, + decode_freq_penalty_beta=args.decode_freq_penalty_beta, + decode_freq_penalty_floor=args.decode_freq_penalty_floor, + decode_freq_penalty_start=args.decode_freq_penalty_start, + decode_freq_penalty_end=args.decode_freq_penalty_end, + decode_freq_penalty_power=args.decode_freq_penalty_power, + start_t=args.start_t, + start_init=args.start_init, + final_sample_mode=args.final_sample_mode, + final_sample_temp=args.final_sample_temp, + final_top_k=args.final_top_k, + final_top_p=args.final_top_p, + final_freq_penalty_alpha=args.final_freq_penalty_alpha, + final_freq_penalty_beta=args.final_freq_penalty_beta, + final_freq_penalty_floor=args.final_freq_penalty_floor, + eps=args.eps, + ) + + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + summary_path = out_dir / "summary.jsonl" + samples_path = out_dir / "samples.txt" + decoded_cache = [] + with summary_path.open("w", encoding="utf-8") as sf, samples_path.open("w", encoding="utf-8") as tf: + for setting in [ + DecodeSetting(temp, final) + for temp in parse_temps(args.endpoint_temps) + for final in parse_final_from(args.final_from) + ]: + torch.manual_seed(args.seed) + ids, raw_texts = decode_blockar_samples( + model, + tokenizer, + setting, + cfg, + n_samples=args.n_samples, + batch_size=args.decode_batch, + decode_time_grid=decode_time_grid, + device=device, + ) + stripped = [strip_common_special(t) for t in raw_texts] + decoded_cache.append((setting, ids, raw_texts, stripped)) + sf.write(json.dumps({"type": "decode_done", "step": step, "setting": asdict(setting)}, ensure_ascii=False) + "\n") + for i in range(min(args.save_samples, len(raw_texts))): + tf.write(f"===== temp={setting.endpoint_temp} final={setting.final_from} sample={i} =====\n") + tf.write(stripped[i] + "\n\n") + sf.flush() + tf.flush() + + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + scorer_tok = AutoTokenizer.from_pretrained(args.scorer) + if scorer_tok.pad_token_id is None: + scorer_tok.pad_token = scorer_tok.eos_token + scorer_tok.pad_token_id = scorer_tok.eos_token_id + scorer = AutoModelForCausalLM.from_pretrained(args.scorer).to(device).eval() + if getattr(scorer.config, "pad_token_id", None) is None: + scorer.config.pad_token_id = scorer_tok.pad_token_id + + with summary_path.open("a", encoding="utf-8") as sf: + for setting, ids, raw_texts, stripped_texts in decoded_cache: + kept_raw, _ = filter_generated_texts(raw_texts, min_chars=1, normalize_whitespace=False, drop_empty=True) + kept_stripped, _ = filter_generated_texts(stripped_texts, min_chars=1, normalize_whitespace=True, drop_empty=True) + raw_ppl = score_with_loaded( + kept_raw, + scorer, + scorer_tok, + batch_size=args.score_batch, + max_length=args.score_max_length, + device=device, + ) + stripped_ppl = score_with_loaded( + kept_stripped, + scorer, + scorer_tok, + batch_size=args.score_batch, + max_length=args.score_max_length, + device=device, + ) + diversity = summarize_token_diversity(ids).__dict__ + summary = { + "type": "summary", + "checkpoint": args.checkpoint, + "step": step, + "blockar_decode": True, + "decode": { + **asdict(cfg), + "endpoint_temp": setting.endpoint_temp, + "final_from": setting.final_from, + "decode_time_schedule": args.decode_time_schedule, + "decode_time_grid": decode_time_grid, + "n_samples": args.n_samples, + "seed": args.seed, + }, + "raw_genppl": raw_ppl, + "stripped_genppl": stripped_ppl, + "diversity": diversity, + } + sf.write(json.dumps(summary, ensure_ascii=False) + "\n") + sf.flush() + print("[summary]", json.dumps(summary, ensure_ascii=False), flush=True) + print(f"[done] {out_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/eval_lm1b_200k_methods_genppl_20260506.py b/LTA_openwebtext_dualt/scripts/eval_lm1b_200k_methods_genppl_20260506.py new file mode 100644 index 0000000000000000000000000000000000000000..89cd4270c8b20d88e358f0ba0e6dc6cbe584184a --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/eval_lm1b_200k_methods_genppl_20260506.py @@ -0,0 +1,750 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from flowtext_lab.ar_model import CausalTransformerLM +from flowtext_lab.baseline_losses import ( + BaselineLossConfig, + _duo_gamma_to_alpha, + _duo_gumbel_tau_inverse, + load_duo_integral_cache, +) +from flowtext_lab.decode import sample_noise_simplex, state_for_model +from flowtext_lab.genppl import filter_generated_texts, summarize_token_diversity +from flowtext_lab.model import EndpointPredictor +from flowtext_lab.tokenization import BpeTextTokenizer + + +SPECIAL_RE = re.compile(r"\s+") + + +@dataclass(frozen=True) +class EvalTarget: + name: str + kind: str + checkpoint: str + + +TARGETS = [ + EvalTarget( + "mdlm_200k", + "mdlm", + "runs/lm1b_mdlm_unified_ddit_small_len128_gbs512_8gpu_1m_20260505_repro/step_0200000.pt", + ), + EvalTarget( + "duo_200k", + "duo", + "runs/lm1b_duo_unified_ddit_small_len128_gbs512_8gpu_1m_20260505_repro/step_0200000.pt", + ), + EvalTarget( + "categorical_fullvocab_200k", + "categorical_fullvocab", + "runs/lta_lm1b_dirichlet_categorical_fullvocab_dualt_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0/step_0200000.pt", + ), + EvalTarget( + "ar_rowshard_200k", + "ar", + "runs/ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m_rowshard_b64_resume4000_20260504_203021/step_0200000.pt", + ), +] + + +def strip_common_special(text: str) -> str: + text = text.replace("[CLS]", " ").replace("[SEP]", " ").replace("[PAD]", " ") + text = text.replace("<|endoftext|>", " ") + return SPECIAL_RE.sub(" ", text).strip() + + +def write_raw_samples(path: Path, title: str, summary: dict, texts: list[str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + f.write(title + "\n") + f.write(json.dumps(summary, ensure_ascii=False, indent=2) + "\n\n") + for i, text in enumerate(texts): + f.write(f"===== sample {i:03d} =====\n") + f.write(text + "\n\n") + + +def build_endpoint_model(ckpt: dict, tokenizer: BpeTextTokenizer, kind: str, device: torch.device) -> EndpointPredictor: + train_args = ckpt.get("args", {}) + token_vocab = tokenizer.vocab_size + train_vocab = int(ckpt.get("train_vocab_size") or ckpt.get("vocab_size") or token_vocab) + if kind == "mdlm": + train_vocab = max(train_vocab, token_vocab + 1) + input_format = str(train_args.get("state_format", train_args.get("input_format", "prob"))) + model = EndpointPredictor( + vocab_size=train_vocab, + max_len=int(train_args.get("max_len", 128)), + d_model=int(train_args.get("d_model", 768)), + n_heads=int(train_args.get("n_heads", 12)), + n_layers=int(train_args.get("n_layers", 12)), + dim_ff=int(train_args.get("dim_ff", 3072)), + dropout=0.0, + model_type=str(train_args.get("model_type", "ddit")), + cond_dim=int(train_args.get("cond_dim", 128)), + input_format=input_format, + ).to(device) + model.load_state_dict(ckpt["model"], strict=True) + model.eval() + return model + + +def build_ar_model(ckpt: dict, tokenizer: BpeTextTokenizer, device: torch.device) -> CausalTransformerLM: + train_args = ckpt.get("args", {}) + model = CausalTransformerLM( + vocab_size=tokenizer.vocab_size, + max_len=int(train_args.get("max_len", 128)), + d_model=int(train_args.get("d_model", 768)), + n_heads=int(train_args.get("n_heads", 12)), + n_layers=int(train_args.get("n_layers", 12)), + dim_ff=int(train_args.get("dim_ff", 3072)), + dropout=0.0, + tie_embeddings=bool(train_args.get("tie_embeddings", True)), + ).to(device) + model.load_state_dict(ckpt["model"], strict=True) + model.eval() + return model + + +@torch.inference_mode() +def decode_lta_categorical( + model: EndpointPredictor, + tokenizer: BpeTextTokenizer, + *, + n_samples: int, + batch_size: int, + max_len: int, + steps: int, + seed: int, + device: torch.device, +) -> tuple[list[list[int]], list[str], dict]: + torch.manual_seed(seed) + eps = 1e-8 + concentration_min = 1.0 + concentration_max = 1024.0 + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + remaining = n_samples + while remaining > 0: + bs = min(batch_size, remaining) + probs = sample_noise_simplex( + (bs, max_len), + tokenizer.vocab_size, + device, + eps, + noise_mode="dirichlet", + target_prob=1.0, + noise_sigma=-1.0, + dirichlet_concentration=1.0, + ) + attn = torch.ones((bs, max_len), dtype=torch.bool, device=device) + last_endpoint = probs + for step in range(steps): + t = torch.full((bs,), 0.5, dtype=torch.float32, device=device) + logits = model(state_for_model(model, probs, eps), t, attn).float() / 1.3 + endpoint = F.softmax(logits, dim=-1) + last_endpoint = endpoint + support_t = (step + 1) / max(steps, 1) + semantic_t = support_t ** 1.5 + anchor = probs.clamp_min(eps) + anchor = anchor / anchor.sum(dim=-1, keepdim=True).clamp_min(eps) + forward_endpoint = (1.0 - semantic_t) * anchor + semantic_t * endpoint + forward_endpoint = forward_endpoint.clamp_min(eps) + forward_endpoint = forward_endpoint / forward_endpoint.sum(dim=-1, keepdim=True).clamp_min(eps) + mean = (1.0 - support_t) / float(tokenizer.vocab_size) + support_t * forward_endpoint + mean = mean.clamp_min(eps) + mean = mean / mean.sum(dim=-1, keepdim=True).clamp_min(eps) + log_min = math.log(concentration_min) + log_max = math.log(concentration_max) + conc = math.exp(log_min + support_t * (log_max - log_min)) + alpha = (mean * conc).clamp_min(eps) + probs = torch._standard_gamma(alpha).clamp_min(eps) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(eps) + final_probs = 0.5 * probs + 0.5 * last_endpoint + ids = final_probs.argmax(dim=-1).detach().cpu().tolist() + all_ids.extend(ids) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in ids) + remaining -= bs + print(f"[categorical] generated {n_samples - remaining}/{n_samples}", flush=True) + decode = { + "kind": "categorical_fullvocab", + "steps": steps, + "model_t_mode": "const05", + "decode_rule": "dual_line_resample", + "support_power": 1.0, + "semantic_power": 1.5, + "anchor_mode": "state", + "noise_init": "dirichlet", + "dirichlet_concentration": 1.0, + "concentration_min": concentration_min, + "concentration_max": concentration_max, + "endpoint_temp": 1.3, + "final_from": "blend", + "n_samples": n_samples, + "seed": seed, + } + return all_ids, all_texts, decode + + +@torch.inference_mode() +def decode_dense_baseline( + model: EndpointPredictor, + tokenizer: BpeTextTokenizer, + *, + kind: str, + n_samples: int, + batch_size: int, + max_len: int, + steps: int, + seed: int, + device: torch.device, +) -> tuple[list[list[int]], list[str], dict]: + torch.manual_seed(seed) + eps = 1e-8 + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + remaining = n_samples + while remaining > 0: + bs = min(batch_size, remaining) + probs = sample_noise_simplex( + (bs, max_len), + tokenizer.vocab_size, + device, + eps, + noise_mode="dirichlet", + target_prob=1.0, + noise_sigma=-1.0, + dirichlet_concentration=1.0, + ) + attn = torch.ones((bs, max_len), dtype=torch.bool, device=device) + last_logits = None + for step in range(steps): + progress = (step + 1) / max(steps, 1) + sigma = -math.log(max(progress, 1e-3)) + t = torch.full((bs,), sigma, dtype=torch.float32, device=device) + logits = model(state_for_model(model, probs, eps), t, attn).float() + logits = logits[..., : tokenizer.vocab_size] + last_logits = logits + endpoint = F.softmax(logits, dim=-1) + s = step / max(steps, 1) + t_next = (step + 1) / max(steps, 1) + gamma = min((t_next - s) / max(1.0 - s, eps), 1.0) + probs = probs + gamma * (endpoint - probs) + probs = probs.clamp_min(eps) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(eps) + final = probs if last_logits is None else 0.5 * probs + 0.5 * F.softmax(last_logits, dim=-1) + ids = final.argmax(dim=-1).detach().cpu().tolist() + all_ids.extend(ids) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in ids) + remaining -= bs + print(f"[{kind}] generated {n_samples - remaining}/{n_samples}", flush=True) + decode = { + "kind": kind, + "steps": steps, + "decode_rule": "sigma_flowmap_dense", + "noise_init": "dirichlet", + "dirichlet_concentration": 1.0, + "final_from": "blend", + "n_samples": n_samples, + "seed": seed, + } + return all_ids, all_texts, decode + + +def _sample_from_logits(logits: torch.Tensor, temp: float) -> torch.Tensor: + if temp <= 0: + return logits.argmax(dim=-1) + probs = F.softmax(logits.float() / float(temp), dim=-1) + return torch.multinomial(probs.reshape(-1, probs.size(-1)), num_samples=1).view(*probs.shape[:-1]) + + +@torch.inference_mode() +def decode_duo_relaxed( + model: EndpointPredictor, + tokenizer: BpeTextTokenizer, + ckpt: dict, + *, + n_samples: int, + batch_size: int, + max_len: int, + steps: int, + temp: float, + seed: int, + device: torch.device, +) -> tuple[list[list[int]], list[str], dict]: + """Approximate DUO reverse process during the relaxed/Gaussian curriculum. + + The DUO checkpoint at 200k is still before duo_curriculum_end, so training + input is not a simplex probability state. It is a Gaussian-relaxed one-hot + state scaled by the curriculum inverse temperature. This sampler keeps the + model on that support: predict x0, sample ids, then re-noise those ids at + the next cleaner gamma value. + """ + torch.manual_seed(seed) + train_args = ckpt.get("args", {}) + ckpt_step = int(ckpt.get("step", 0) or 0) + cfg = BaselineLossConfig( + method="duo", + sampling_eps=float(train_args.get("sampling_eps", 1e-3)), + duo_integral_cache_path=str(train_args.get("duo_integral_cache_path", "")), + duo_gamma_min=float(train_args.get("duo_gamma_min", -3.5)), + duo_gamma_max=float(train_args.get("duo_gamma_max", -1.75)), + duo_gumbel_tau_log10_start=float(train_args.get("duo_gumbel_tau_log10_start", -3.0)), + duo_gumbel_tau_log10_end=float(train_args.get("duo_gumbel_tau_log10_end", -3.0)), + duo_curriculum_start=int(train_args.get("duo_curriculum_start", 0)), + duo_curriculum_end=int(train_args.get("duo_curriculum_end", 500_000)), + duo_finite_diff_T=int(train_args.get("duo_finite_diff_T", 1000)), + ) + cache = load_duo_integral_cache(cfg.duo_integral_cache_path, device) + tau_inv = float(_duo_gumbel_tau_inverse(ckpt_step, cfg)) + vocab_size = tokenizer.vocab_size + gamma_noisy = float(cfg.duo_gamma_max) + gamma_clean = float(cfg.duo_gamma_min) + + def gamma_at(frac: float) -> torch.Tensor: + value = gamma_noisy + (gamma_clean - gamma_noisy) * float(frac) + return torch.tensor(value, dtype=torch.float32, device=device) + + def sigma_from_gamma(gamma: torch.Tensor, bs: int) -> torch.Tensor: + alpha = _duo_gamma_to_alpha(gamma.view(1), cache)[0] + return (-torch.log(alpha.clamp_min(1e-8))).expand(bs) + + def relaxed_state(ids: torch.Tensor, gamma: torch.Tensor) -> torch.Tensor: + one_hot = F.one_hot(ids, vocab_size).float() + alpha_g = torch.sigmoid(-gamma).sqrt() + sigma_g = torch.sigmoid(gamma).sqrt() + noise = torch.randn_like(one_hot) + return (alpha_g * one_hot + sigma_g * noise) * tau_inv + + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + remaining = n_samples + while remaining > 0: + bs = min(batch_size, remaining) + attn = torch.ones((bs, max_len), dtype=torch.bool, device=device) + ids = torch.randint(0, vocab_size, (bs, max_len), dtype=torch.long, device=device) + state = relaxed_state(ids, gamma_at(0.0)) + for step in range(steps): + frac = step / max(steps - 1, 1) + gamma = gamma_at(frac) + logits = model(state, sigma_from_gamma(gamma, bs), attn).float()[..., :vocab_size] + ids = _sample_from_logits(logits, temp) + next_frac = (step + 1) / max(steps, 1) + state = relaxed_state(ids, gamma_at(next_frac)) + logits = model(state, sigma_from_gamma(gamma_at(1.0), bs), attn).float()[..., :vocab_size] + ids = _sample_from_logits(logits, temp) + rows = ids.detach().cpu().tolist() + all_ids.extend(rows) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in rows) + remaining -= bs + print(f"[duo relaxed] generated {n_samples - remaining}/{n_samples}", flush=True) + decode = { + "kind": "duo", + "steps": steps, + "decode_rule": "relaxed_gaussian_renoise", + "gamma_min": gamma_clean, + "gamma_max": gamma_noisy, + "tau_inverse": tau_inv, + "temp": temp, + "n_samples": n_samples, + "seed": seed, + } + return all_ids, all_texts, decode + + +@torch.inference_mode() +def decode_mdlm( + model: EndpointPredictor, + tokenizer: BpeTextTokenizer, + *, + n_samples: int, + batch_size: int, + max_len: int, + steps: int, + temp: float, + selection_noise: float, + lock_bos_eos: bool, + time_from_current_mask: bool, + seed: int, + device: torch.device, +) -> tuple[list[list[int]], list[str], dict]: + torch.manual_seed(seed) + mask_id = tokenizer.vocab_size + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + remaining = n_samples + while remaining > 0: + bs = min(batch_size, remaining) + ids = torch.full((bs, max_len), mask_id, dtype=torch.long, device=device) + attn = torch.ones((bs, max_len), dtype=torch.bool, device=device) + filled = torch.zeros((bs, max_len), dtype=torch.bool, device=device) + if lock_bos_eos: + ids[:, 0] = tokenizer.bos_id + ids[:, -1] = tokenizer.eos_id + filled[:, 0] = True + filled[:, -1] = True + locked_tokens = int(filled[0].sum().item()) if lock_bos_eos else 0 + free_tokens = max_len - locked_tokens + for step in range(steps): + progress = (step + 1) / max(steps, 1) + if time_from_current_mask: + clean_frac = filled.float().mean(dim=1).clamp(1e-3, 1.0) + t = -torch.log(clean_frac).to(dtype=torch.float32) + else: + sigma = -math.log(max(progress, 1e-3)) + t = torch.full((bs,), sigma, dtype=torch.float32, device=device) + logits = model(ids, t, attn).float()[..., : tokenizer.vocab_size] + probs = F.softmax(logits, dim=-1) + if temp <= 0: + conf, pred = probs.max(dim=-1) + else: + sample_probs = F.softmax(logits / float(temp), dim=-1) + pred = torch.multinomial( + sample_probs.reshape(-1, sample_probs.size(-1)), + num_samples=1, + ).view(bs, max_len) + conf = torch.gather(probs, -1, pred.unsqueeze(-1)).squeeze(-1) + if selection_noise > 0: + u = torch.rand_like(conf).clamp_(1e-6, 1.0 - 1e-6) + conf = conf + float(selection_noise) * (-torch.log(-torch.log(u))) + target_filled = locked_tokens + int(math.ceil(progress * free_tokens)) + for b in range(bs): + need = max(0, min(max_len, target_filled) - int(filled[b].sum().item())) + if step == steps - 1: + need = int((~filled[b]).sum().item()) + if need <= 0: + continue + scores = conf[b].masked_fill(filled[b], -1.0) + choose = torch.topk(scores, k=need).indices + ids[b, choose] = pred[b, choose] + filled[b, choose] = True + out = ids.clamp_max(tokenizer.vocab_size - 1).detach().cpu().tolist() + all_ids.extend(out) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in out) + remaining -= bs + print(f"[mdlm] generated {n_samples - remaining}/{n_samples}", flush=True) + decode = { + "kind": "mdlm", + "steps": steps, + "decode_rule": "stochastic_confidence_unmask" if temp > 0 else "argmax_confidence_unmask", + "start": "all_mask", + "temp": temp, + "selection_noise": selection_noise, + "lock_bos_eos": lock_bos_eos, + "time_from_current_mask": time_from_current_mask, + "n_samples": n_samples, + "seed": seed, + } + return all_ids, all_texts, decode + + +@torch.inference_mode() +def decode_ar( + model: CausalTransformerLM, + tokenizer: BpeTextTokenizer, + *, + temp: float, + n_samples: int, + batch_size: int, + max_len: int, + seed: int, + device: torch.device, +) -> tuple[list[list[int]], list[str], dict]: + torch.manual_seed(seed) + all_ids: list[list[int]] = [] + all_texts: list[str] = [] + remaining = n_samples + while remaining > 0: + bs = min(batch_size, remaining) + ids = torch.full((bs, 1), tokenizer.bos_id, dtype=torch.long, device=device) + for _ in range(max_len - 1): + ctx = ids[:, -max_len:] + logits = model(ctx, torch.ones_like(ctx, dtype=torch.bool))[:, -1].float() + if temp <= 0: + nxt = logits.argmax(dim=-1, keepdim=True) + else: + probs = F.softmax(logits / temp, dim=-1) + nxt = torch.multinomial(probs, num_samples=1) + ids = torch.cat([ids, nxt], dim=1) + rows = ids.detach().cpu().tolist() + all_ids.extend(rows) + all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in rows) + remaining -= bs + print(f"[ar temp={temp:g}] generated {n_samples - remaining}/{n_samples}", flush=True) + decode = { + "kind": "ar_sample", + "temp": temp, + "max_new_tokens": max_len - 1, + "n_samples": n_samples, + "seed": seed, + } + return all_ids, all_texts, decode + + +@torch.no_grad() +def score_with_loaded( + texts: list[str], + scorer, + scorer_tok, + *, + batch_size: int, + max_length: int, + device: torch.device, +) -> dict: + total_nll = 0.0 + total_tokens = 0 + skipped = 0 + for start in range(0, len(texts), batch_size): + batch_texts = texts[start : start + batch_size] + enc = scorer_tok( + batch_texts, + return_tensors="pt", + return_token_type_ids=False, + return_attention_mask=True, + padding=True, + truncation=True, + max_length=max_length, + ).to(device) + input_ids = enc["input_ids"] + attention_mask = enc["attention_mask"] + if input_ids.size(1) < 2: + skipped += len(batch_texts) + continue + logits = scorer(input_ids=input_ids, attention_mask=attention_mask).logits.transpose(-1, -2) + token_nll = F.cross_entropy(logits[..., :-1].float(), input_ids[..., 1:], reduction="none") + if scorer_tok.eos_token_id is not None: + first_eos = (input_ids == scorer_tok.eos_token_id).cumsum(-1) == 1 + token_mask = input_ids != scorer_tok.eos_token_id + shift_mask = (first_eos[..., 1:] | token_mask[..., 1:]).contiguous() + else: + shift_mask = attention_mask[..., 1:].contiguous().bool() + total_nll += float(token_nll[shift_mask].sum().detach().cpu()) + total_tokens += int(shift_mask.sum().detach().cpu()) + nll = total_nll / max(total_tokens, 1) + return { + "ppl": float(math.exp(min(nll, 50.0))), + "nll_per_token": float(nll), + "tokens": total_tokens, + "kept_samples": len(texts), + "total_samples": len(texts), + "empty_rate": 0.0, + "skipped_samples": skipped, + } + + +def score_and_write( + *, + target: EvalTarget, + checkpoint: str, + step: int | None, + decode: dict, + ids: list[list[int]], + raw_texts: list[str], + scorer, + scorer_tok, + score_batch: int, + score_max_length: int, + device: torch.device, + out_dir: Path, +) -> dict: + stripped = [strip_common_special(t) for t in raw_texts] + kept_raw, _ = filter_generated_texts(raw_texts, min_chars=1, normalize_whitespace=False, drop_empty=True) + kept_stripped, _ = filter_generated_texts(stripped, min_chars=1, normalize_whitespace=True, drop_empty=True) + raw_ppl = score_with_loaded(kept_raw, scorer, scorer_tok, batch_size=score_batch, max_length=score_max_length, device=device) + stripped_ppl = score_with_loaded( + kept_stripped, scorer, scorer_tok, batch_size=score_batch, max_length=score_max_length, device=device + ) + diversity = summarize_token_diversity(ids).__dict__ + summary = { + "type": "summary", + "name": target.name, + "kind": target.kind, + "checkpoint": checkpoint, + "step": step, + "decode": decode, + "raw_genppl": raw_ppl, + "stripped_genppl": stripped_ppl, + "diversity": diversity, + } + jsonl_path = out_dir / f"{target.name}_scored.jsonl" + with jsonl_path.open("w", encoding="utf-8") as f: + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + for i, (raw, clean) in enumerate(zip(raw_texts, stripped)): + f.write( + json.dumps( + { + "type": "sample", + "name": target.name, + "index": i, + "raw_text": raw, + "stripped_text": clean, + }, + ensure_ascii=False, + ) + + "\n" + ) + write_raw_samples(out_dir / f"{target.name}_raw_samples.txt", target.name, summary, raw_texts) + return summary + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--tokenizer_path", required=True) + p.add_argument("--scorer", required=True) + p.add_argument("--out_dir", required=True) + p.add_argument("--only", default="", help="Comma-separated target names/kinds.") + p.add_argument("--n_samples", type=int, default=256) + p.add_argument("--max_len", type=int, default=128) + p.add_argument("--decode_batch", type=int, default=32) + p.add_argument("--score_batch", type=int, default=8) + p.add_argument("--score_max_length", type=int, default=256) + p.add_argument("--steps", type=int, default=1024) + p.add_argument("--baseline_steps", type=int, default=256) + p.add_argument("--seed", type=int, default=20260506) + p.add_argument("--ar_temps", default="0.8,1.0") + p.add_argument("--duo_temp", type=float, default=1.0) + p.add_argument("--mdlm_temp", type=float, default=1.0) + p.add_argument("--mdlm_selection_noise", type=float, default=0.0) + p.add_argument("--mdlm_lock_bos_eos", action="store_true") + p.add_argument("--mdlm_time_from_current_mask", action="store_true") + args = p.parse_args() + + selected = {x.strip() for x in args.only.split(",") if x.strip()} + targets = [t for t in TARGETS if not selected or t.name in selected or t.kind in selected] + if not targets: + raise SystemExit(f"No targets selected by --only={args.only!r}") + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + summary_path = out_dir / "summary.jsonl" + summary_path.write_text("", encoding="utf-8") + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path) + + for target in targets: + ckpt = torch.load(target.checkpoint, map_location="cpu", weights_only=False) + step = ckpt.get("step") + print(f"[load] {target.name} step={step} ckpt={target.checkpoint}", flush=True) + generated: list[tuple[EvalTarget, str, int | None, dict, list[list[int]], list[str]]] = [] + if target.kind == "ar": + model = build_ar_model(ckpt, tokenizer, device) + for temp in [float(x) for x in args.ar_temps.split(",") if x.strip()]: + ids, texts, decode = decode_ar( + model, + tokenizer, + temp=temp, + n_samples=args.n_samples, + batch_size=args.decode_batch, + max_len=args.max_len, + seed=args.seed, + device=device, + ) + ar_target = EvalTarget(f"{target.name}_t{str(temp).replace('.', 'p')}", target.kind, target.checkpoint) + generated.append((ar_target, target.checkpoint, step, decode, ids, texts)) + else: + model = build_endpoint_model(ckpt, tokenizer, target.kind, device) + if target.kind == "categorical_fullvocab": + ids, texts, decode = decode_lta_categorical( + model, + tokenizer, + n_samples=args.n_samples, + batch_size=args.decode_batch, + max_len=args.max_len, + steps=args.steps, + seed=args.seed, + device=device, + ) + elif target.kind == "mdlm": + ids, texts, decode = decode_mdlm( + model, + tokenizer, + n_samples=args.n_samples, + batch_size=args.decode_batch, + max_len=args.max_len, + steps=args.baseline_steps, + temp=args.mdlm_temp, + selection_noise=args.mdlm_selection_noise, + lock_bos_eos=args.mdlm_lock_bos_eos, + time_from_current_mask=args.mdlm_time_from_current_mask, + seed=args.seed, + device=device, + ) + elif target.kind == "duo": + ids, texts, decode = decode_duo_relaxed( + model, + tokenizer, + ckpt, + n_samples=args.n_samples, + batch_size=args.decode_batch, + max_len=args.max_len, + steps=args.baseline_steps, + temp=args.duo_temp, + seed=args.seed, + device=device, + ) + else: + ids, texts, decode = decode_dense_baseline( + model, + tokenizer, + kind=target.kind, + n_samples=args.n_samples, + batch_size=args.decode_batch, + max_len=args.max_len, + steps=args.baseline_steps, + seed=args.seed, + device=device, + ) + generated.append((target, target.checkpoint, step, decode, ids, texts)) + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + scorer_tok = AutoTokenizer.from_pretrained(args.scorer) + if scorer_tok.pad_token_id is None: + scorer_tok.pad_token = scorer_tok.eos_token + scorer_tok.pad_token_id = scorer_tok.eos_token_id + scorer = AutoModelForCausalLM.from_pretrained(args.scorer).to(device).eval() + if getattr(scorer.config, "pad_token_id", None) is None: + scorer.config.pad_token_id = scorer_tok.pad_token_id + with summary_path.open("a", encoding="utf-8") as sf: + for eval_target, checkpoint, eval_step, decode, ids, texts in generated: + summary = score_and_write( + target=eval_target, + checkpoint=checkpoint, + step=eval_step, + decode=decode, + ids=ids, + raw_texts=texts, + scorer=scorer, + scorer_tok=scorer_tok, + score_batch=args.score_batch, + score_max_length=args.score_max_length, + device=device, + out_dir=out_dir, + ) + sf.write(json.dumps(summary, ensure_ascii=False) + "\n") + sf.flush() + print("[summary]", json.dumps(summary, ensure_ascii=False), flush=True) + del scorer, scorer_tok, generated + if torch.cuda.is_available(): + torch.cuda.empty_cache() + print(f"[done] {out_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/eval_train8_decode_acc.py b/LTA_openwebtext_dualt/scripts/eval_train8_decode_acc.py new file mode 100644 index 0000000000000000000000000000000000000000..661d3f043a8eecdafd6f5bdb0ebd1198550f35f5 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/eval_train8_decode_acc.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +import numpy as np +import torch + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) +SCRIPTS_DIR = REPO_ROOT / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from infer_context_compare_from_c128 import build_model, decode # noqa: E402 +from flowtext_lab.tokenization import BpeTextTokenizer # noqa: E402 + + +def ckpt_step(path: Path) -> int: + m = re.search(r"step_(\d+)\.pt$", path.name) + if not m: + return -1 + return int(m.group(1)) + + +def select_ckpts(run_dir: Path, *, latest_only: bool, step_stride: int) -> list[Path]: + ckpts = sorted(run_dir.glob("step_*.pt"), key=ckpt_step) + if not ckpts: + return [] + if latest_only: + return [ckpts[-1]] + if step_stride > 0: + picked = [p for p in ckpts if ckpt_step(p) % step_stride == 0] + if ckpts[-1] not in picked: + picked.append(ckpts[-1]) + return sorted(set(picked), key=ckpt_step) + return ckpts + + +def load_refs(data_dir: Path, max_len: int) -> np.ndarray: + meta = json.loads((data_dir / "meta.json").read_text()) + n = int(meta.get("num_chunks", meta.get("n_chunks", 0))) + if n <= 0: + size = (data_dir / "chunks.i32.bin").stat().st_size // np.dtype(np.int32).itemsize + n = size // max_len + arr = np.memmap(data_dir / "chunks.i32.bin", dtype=np.int32, mode="r") + arr = np.asarray(arr).reshape(n, -1) + return arr[:, :max_len].copy() + + +def token_match_metrics(ids: list[list[int]], refs: np.ndarray) -> dict[str, object]: + gen = np.asarray(ids, dtype=np.int32) + if gen.ndim != 2: + raise ValueError(f"expected 2D generated ids, got {gen.shape}") + if gen.shape[1] != refs.shape[1]: + n = min(gen.shape[1], refs.shape[1]) + gen = gen[:, :n] + refs = refs[:, :n] + matches = (gen[:, None, :] == refs[None, :, :]).mean(axis=2) + best_idx = matches.argmax(axis=1) + best_acc = matches[np.arange(matches.shape[0]), best_idx] + exact = best_acc >= 1.0 + exact_ref_hits = sorted(set(best_idx[exact].astype(int).tolist())) + return { + "n_gen": int(gen.shape[0]), + "n_refs": int(refs.shape[0]), + "token_acc_mean": float(best_acc.mean()), + "token_acc_min": float(best_acc.min()), + "token_acc_max": float(best_acc.max()), + "exact_acc": float(exact.mean()), + "exact_count": int(exact.sum()), + "exact_ref_coverage": float(len(exact_ref_hits) / max(refs.shape[0], 1)), + "exact_ref_count": int(len(exact_ref_hits)), + "exact_ref_hits": exact_ref_hits, + "best_ref_idx": best_idx.astype(int).tolist(), + "best_token_acc": best_acc.astype(float).tolist(), + } + + +@torch.inference_mode() +def eval_one( + ckpt_path: Path, + tokenizer: BpeTextTokenizer, + refs: np.ndarray, + args: argparse.Namespace, + endpoint_softening: str, + device: torch.device, +) -> dict[str, object]: + ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False, mmap=True) + model = build_model(ckpt, tokenizer, args.max_len, device, args.pos_extend) + ids, _texts, _traces = decode( + model, + tokenizer, + max_len=args.max_len, + n_samples=args.n_samples, + batch_size=args.batch_size, + steps=args.steps, + seed=args.seed + ckpt_step(ckpt_path) + args.max_len, + device=device, + decode_rule=args.decode_rule, + support_power=args.support_power, + semantic_power=args.semantic_power, + early_temp=args.early_temp, + late_temp=args.late_temp, + temp_end=args.temp_end, + temp_power=args.temp_power, + hybrid_switch=args.hybrid_switch, + tail_temp=args.tail_temp, + c_min=args.c_min, + c_max=args.c_max, + model_t_mode=args.model_t_mode, + time_schedule=args.time_schedule, + time_logit_mean=args.time_logit_mean, + time_logit_std=args.time_logit_std, + time_power=args.time_power, + input_noise_scale=args.input_noise_scale, + input_noise_until=args.input_noise_until, + input_noise_dirichlet_concentration=args.input_noise_dirichlet_concentration, + endpoint_softening=endpoint_softening, + endpoint_soft_power=args.endpoint_soft_power, + endpoint_soft_min_conf=args.endpoint_soft_min_conf, + endpoint_soft_max_conf=args.endpoint_soft_max_conf, + final_from=args.final_from, + final_decode=args.final_decode, + final_sample_temp=args.final_sample_temp, + final_top_k=args.final_top_k, + final_top_p=args.final_top_p, + eps=1e-8, + fixed_first_token_id=None, + fixed_first_initial_argmax=False, + ) + metrics = token_match_metrics(ids, refs) + del model + if device.type == "cuda": + torch.cuda.empty_cache() + return { + "run": ckpt_path.parent.name, + "checkpoint": str(ckpt_path), + "ckpt_step": ckpt_step(ckpt_path), + "endpoint_softening": endpoint_softening, + "decode_rule": args.decode_rule, + "steps": args.steps, + "time_schedule": args.time_schedule, + "model_t_mode": args.model_t_mode, + "final_from": args.final_from, + **metrics, + } + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--runs_glob", default="runs/train8_n1024_*") + ap.add_argument("--data_dir", required=True) + ap.add_argument("--tokenizer_path", required=True) + ap.add_argument("--out_dir", required=True) + ap.add_argument("--max_len", type=int, default=1024) + ap.add_argument("--n_samples", type=int, default=64) + ap.add_argument("--batch_size", type=int, default=2) + ap.add_argument("--latest_only", action="store_true") + ap.add_argument("--step_stride", type=int, default=100) + ap.add_argument("--endpoint_softenings", default="none") + ap.add_argument("--steps", type=int, default=128) + ap.add_argument("--decode_rule", default="flowmap") + ap.add_argument("--support_power", type=float, default=1.0) + ap.add_argument("--semantic_power", type=float, default=1.0) + ap.add_argument("--early_temp", type=float, default=1.0) + ap.add_argument("--late_temp", type=float, default=1.0) + ap.add_argument("--temp_end", type=float, default=1.0) + ap.add_argument("--temp_power", type=float, default=1.0) + ap.add_argument("--hybrid_switch", type=float, default=0.5) + ap.add_argument("--tail_temp", type=float, default=-1.0) + ap.add_argument("--c_min", type=float, default=1.0) + ap.add_argument("--c_max", type=float, default=512.0) + ap.add_argument("--model_t_mode", default="post") + ap.add_argument("--time_schedule", default="logit_normal") + ap.add_argument("--time_logit_mean", type=float, default=-1.5) + ap.add_argument("--time_logit_std", type=float, default=0.8) + ap.add_argument("--time_power", type=float, default=2.0) + ap.add_argument("--input_noise_scale", type=float, default=0.0) + ap.add_argument("--input_noise_until", type=float, default=1.0) + ap.add_argument("--input_noise_dirichlet_concentration", type=float, default=1.0) + ap.add_argument("--endpoint_soft_power", type=float, default=1.0) + ap.add_argument("--endpoint_soft_min_conf", type=float, default=0.0) + ap.add_argument("--endpoint_soft_max_conf", type=float, default=1.0) + ap.add_argument("--final_from", default="state") + ap.add_argument("--final_decode", default="argmax") + ap.add_argument("--final_sample_temp", type=float, default=1.0) + ap.add_argument("--final_top_k", type=int, default=0) + ap.add_argument("--final_top_p", type=float, default=1.0) + ap.add_argument("--pos_extend", default="repeat") + ap.add_argument("--seed", type=int, default=20260517) + args = ap.parse_args() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path) + refs = load_refs(Path(args.data_dir), args.max_len) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + for stale in ["decode_token_acc.jsonl", "decode_token_acc.tsv", "decode_token_acc_summary.json"]: + path = out_dir / stale + if path.exists(): + path.unlink() + + run_dirs = sorted(Path(".").glob(args.runs_glob)) + endpoint_softenings = [x.strip() for x in args.endpoint_softenings.split(",") if x.strip()] + rows: list[dict[str, object]] = [] + for run_dir in run_dirs: + ckpts = select_ckpts(run_dir, latest_only=args.latest_only, step_stride=args.step_stride) + for ckpt_path in ckpts: + for soft in endpoint_softenings: + print(f"[eval-decode-acc] {run_dir.name} step={ckpt_step(ckpt_path)} soft={soft}", flush=True) + rec = eval_one(ckpt_path, tokenizer, refs, args, soft, device) + rows.append(rec) + with (out_dir / "decode_token_acc.jsonl").open("a", encoding="utf-8") as f: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + + fields = [ + "run", + "ckpt_step", + "endpoint_softening", + "token_acc_mean", + "token_acc_min", + "token_acc_max", + "exact_acc", + "exact_count", + "exact_ref_coverage", + "exact_ref_count", + ] + with (out_dir / "decode_token_acc.tsv").open("w", encoding="utf-8") as f: + f.write("\t".join(fields) + "\n") + for r in rows: + f.write("\t".join(str(r[k]) for k in fields) + "\n") + + best_by_run: dict[str, dict[str, object]] = {} + first_exact_by_run: dict[str, dict[str, object] | None] = {} + for r in rows: + key = f"{r['run']}::{r['endpoint_softening']}" + if key not in best_by_run or float(r["token_acc_mean"]) > float(best_by_run[key]["token_acc_mean"]): + best_by_run[key] = r + if float(r["exact_acc"]) > 0 and key not in first_exact_by_run: + first_exact_by_run[key] = r + summary = { + "num_rows": len(rows), + "best_by_run": best_by_run, + "first_exact_by_run": first_exact_by_run, + } + (out_dir / "decode_token_acc_summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/launch_ar_lm1b_duo_small_4gpu_1m.sh b/LTA_openwebtext_dualt/scripts/launch_ar_lm1b_duo_small_4gpu_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..37876f32acffec23d6f092a732f9523e6a7bd339 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_ar_lm1b_duo_small_4gpu_1m.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +RUN_NAME="${RUN_NAME:-ar_lm1b_flmpack_bert_small_len128_gbs512_4gpu_1m}" +DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-all}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-4}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29641}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-64}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-128}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-20000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +SAMPLE_EVERY="${SAMPLE_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" + +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train_ar.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --sample_every "${SAMPLE_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --tie_embeddings \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1024_fullycoupled_4gpu_small_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1024_fullycoupled_4gpu_small_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..693901705c31ecbcb14d09ef04492287e5326de2 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1024_fullycoupled_4gpu_small_1m.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +# Fully-coupled t ablation: +# model_t == support/Dirichlet t == semantic endpoint t +RUN_NAME="${RUN_NAME:-lta_lm1b_dirichlet_categorical_fullvocab_c1024_fullycoupled_flmpack_onehot_hardce_ddit_small_len128_gbs512_4gpu_1m_nw0}" +DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-all}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-4}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29651}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-64}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-128}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-20000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}" + +if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then + ENABLE_TORCH_COMPILE=0 +fi +if [[ "${DATA_PATH}" == *"lm1b_train_parquet"* && "${NUM_WORKERS}" != "0" ]]; then + echo "LM1B has only 9 parquet shards; forcing NUM_WORKERS=0 to avoid empty DDP dataloader shards." >&2 + NUM_WORKERS=0 +fi + +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" +echo "[launch] method=categorical_fullvocab_c1024_fullycoupled host=$(hostname) time=$(date -Iseconds)" +echo "[launch] cwd=$(pwd)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1_dualt_8gpu_small_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1_dualt_8gpu_small_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..127374db48e71049dffd6f89282ec4ea7427a4e6 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c1_dualt_8gpu_small_1m.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +# C=1 categorical dual-t LM1B, full-vocab wrong-token endpoint. +# Reuse the C_MAX-parametrized 8-GPU launcher so data/tokenizer/model settings +# stay identical to the C=4/C=16/C=64/C=256/C=1024 runs. + +export C_MAX="${C_MAX:-1.0}" +export C_TAG="${C_TAG:-c${C_MAX//./p}}" +export RUN_NAME="${RUN_NAME:-lta_lm1b_dirichlet_categorical_fullvocab_${C_TAG}_dualt_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0}" +export MASTER_PORT="${MASTER_PORT:-29668}" + +exec bash scripts/launch_lta_lm1b_categorical_fullvocab_c64_dualt_8gpu_small_1m.sh diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c4_dualt_8gpu_small_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c4_dualt_8gpu_small_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..5d504595fa42a5062298392b7e3303009a322755 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_c4_dualt_8gpu_small_1m.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +# C=4 categorical dual-t LM1B, full-vocab wrong-token endpoint. +# Reuse the C_MAX-parametrized 8-GPU launcher so data/tokenizer/model settings +# stay identical to the C=64/C=256/C=1024 runs. + +export C_MAX="${C_MAX:-4.0}" +export C_TAG="${C_TAG:-c${C_MAX//./p}}" +export RUN_NAME="${RUN_NAME:-lta_lm1b_dirichlet_categorical_fullvocab_${C_TAG}_dualt_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0}" +export MASTER_PORT="${MASTER_PORT:-29666}" + +exec bash scripts/launch_lta_lm1b_categorical_fullvocab_c64_dualt_8gpu_small_1m.sh diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_dualt_8gpu_duo_small_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_dualt_8gpu_duo_small_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..0b6314fdc55ff32c3b858291ae45b4adb085f935 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_lm1b_categorical_fullvocab_dualt_8gpu_duo_small_1m.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +RUN_NAME="${RUN_NAME:-lta_lm1b_duo_aligned_dirichlet_categorical_fullvocab_dualt_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0}" +DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-all}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29621}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-64}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-128}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-20000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}" +if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then + ENABLE_TORCH_COMPILE=0 +fi +if [[ "${DATA_PATH}" == *"lm1b_train_parquet"* && "${NUM_WORKERS}" != "0" ]]; then + echo "LM1B has only 9 parquet shards; forcing NUM_WORKERS=0 to avoid empty DDP dataloader shards." >&2 + NUM_WORKERS=0 +fi +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode independent \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_8gpu_duo_small_1m.sh.bak_disable_compile_20260429_221413 b/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_8gpu_duo_small_1m.sh.bak_disable_compile_20260429_221413 new file mode 100644 index 0000000000000000000000000000000000000000..1c38997101663e31320c97f0f509cc0c500c3cc3 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_8gpu_duo_small_1m.sh.bak_disable_compile_20260429_221413 @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +# Duo/FLM-repo-aligned engineering defaults: +# - GPT-2 OpenWebText split, train[:-100000], wrapped fixed length 1024 +# - DDiT small: 12 layers, d=768, 12 heads, FF=3072, adaLN-zero +# - global batch 512, per-GPU local batch 32 on 8 GPUs +# - bf16, flash-attn when available, torch.compile max-autotune +# - AdamW lr 3e-4, no weight decay, 2500 warmup, constant schedule +# - Duo-style OWT training budget: 1M optimizer steps +# +# Algorithm remains ours: +# - probability-state LTA +# - Dirichlet bridge +# - hard CE target and one-hot clean anchors/endpoints +# - wrong-token corruption is scheduled by corruption time: p_wrong = 1 - t_corrupt +# - dual-t: model/flow time is separate from corruption/support time + +RUN_NAME="${RUN_NAME:-lta_owt_duo_aligned_dirichlet_dualt_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m}" +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29621}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-1024}" +NUM_WORKERS="${NUM_WORKERS:-4}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-20000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-1}" +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode max-autotune) +fi +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +mkdir -p logs runs "${SAVE_DIR}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + --text_column text \ + --openwebtext_split train_minus_100k \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --eps 1e-8 \ + --infer_steps 1024 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_smoke_1gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_smoke_1gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..90c9a186212c85959b7ccd49bf0cb64646e07200 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_openwebtext_dualt_smoke_1gpu.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +RUN_NAME="${RUN_NAME:-smoke_lta_openwebtext_dirichlet_dualt_tsched_len1024_1gpu}" +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" +GPU="${GPU:-0}" + +mkdir -p logs runs "${SAVE_DIR}" + +CUDA_VISIBLE_DEVICES="${GPU}" python train.py \ + --data_path "${DATA_PATH}" \ + --text_column text \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --max_records 64 \ + --max_len 1024 \ + --batch_size 1 \ + --num_workers 0 \ + --global_batch_size 1 \ + --total_steps 3 \ + --log_every 1 \ + --eval_every 0 \ + --save_every 3 \ + --lr 3e-4 \ + --weight_decay 0 \ + --warmup_steps 3 \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss soft_ce \ + --target_prob 0.99 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --eps 1e-8 \ + --infer_steps 8 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_c1024_fullycoupled_8gpu_len1024_gpt2_cached_chunks_rollout1_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_c1024_fullycoupled_8gpu_len1024_gpt2_cached_chunks_rollout1_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..0c595ad16b9929ae249ed54261c30f1de51af90f --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_c1024_fullycoupled_8gpu_len1024_gpt2_cached_chunks_rollout1_1m.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +# One-step self-rollout training on top of the cached GPT-2 OWT chunk setup. +# Training still supervises the original hard CE target, but with probability +# ROLLOUT_TRAIN_PROB the model first performs one detached flowmap update on +# the current corrupted state, then trains on that self-generated state. + +export OWT_CACHED_CHUNKS=1 +export OWT_CHUNK_CACHE_DIR="${OWT_CHUNK_CACHE_DIR:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k}" +export OWT_CHUNK_CACHE_REBUILD="${OWT_CHUNK_CACHE_REBUILD:-0}" + +export NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +export PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +export TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +export WARMUP_STEPS="${WARMUP_STEPS:-2000}" +export MAX_LEN="${MAX_LEN:-1024}" +export NUM_WORKERS="${NUM_WORKERS:-4}" +export DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-2}" +export LOG_EVERY="${LOG_EVERY:-100}" +export SAVE_EVERY="${SAVE_EVERY:-20000}" +export LATEST_EVERY="${LATEST_EVERY:-1000}" +export EVAL_EVERY="${EVAL_EVERY:-0}" +export ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +export ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" + +export ROLLOUT_TRAIN_PROB="${ROLLOUT_TRAIN_PROB:-1.0}" +export ROLLOUT_TRAIN_STEPS="${ROLLOUT_TRAIN_STEPS:-1}" +export ROLLOUT_TRAIN_INFER_STEPS="${ROLLOUT_TRAIN_INFER_STEPS:-64}" +export ROLLOUT_TRAIN_TEMP="${ROLLOUT_TRAIN_TEMP:-1.45}" +export ROLLOUT_TRAIN_MAX_GAMMA="${ROLLOUT_TRAIN_MAX_GAMMA:-1.0}" +export ROLLOUT_TRAIN_CORRUPT_ONLY="${ROLLOUT_TRAIN_CORRUPT_ONLY:-1}" +export ROLLOUT_TRAIN_SAMPLEWISE="${ROLLOUT_TRAIN_SAMPLEWISE:-0}" +export ROLLOUT_TRAIN_COMPUTE_ALWAYS="${ROLLOUT_TRAIN_COMPUTE_ALWAYS:-0}" + +export DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +export TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" +export TEXT_COLUMN="${TEXT_COLUMN:-text}" +export OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" +export DETOKENIZER="${DETOKENIZER:-auto}" + +export RUN_NAME="${RUN_NAME:-lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_gpt2cached_len1024_rollout1_p${ROLLOUT_TRAIN_PROB//./p}_gbs${GLOBAL_BATCH_SIZE}_${NPROC_PER_NODE}gpu_1m}" + +bash scripts/launch_lta_owt_categorical_fullvocab_c1024_fullycoupled_8gpu_small_1m.sh diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_categorical_fullvocab_c1024_fullycoupled_8gpu_small_1m.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_categorical_fullvocab_c1024_fullycoupled_8gpu_small_1m.sh new file mode 100644 index 0000000000000000000000000000000000000000..0f144c62fc7f286eb1ebfc7e15efd7f568efff47 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_categorical_fullvocab_c1024_fullycoupled_8gpu_small_1m.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +# OpenWebText counterpart of: +# launch_lta_lm1b_categorical_fullvocab_c1024_fullycoupled_8gpu_small_1m.sh +# +# Fully-coupled t ablation: +# model_t == support/Dirichlet t == semantic endpoint t +# +# Important variant switch: +# --categorical_wrong_from_full_vocab +# so the "wrong" token is sampled from all vocab tokens, not vocab \ {gold}. + +C_MAX="${C_MAX:-1024}" +C_TAG="${C_TAG:-c${C_MAX//./p}}" +RUN_NAME="${RUN_NAME:-lta_owt_dirichlet_categorical_fullvocab_${C_TAG}_fullycoupled_flmpack_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m_nw4}" +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" +DETOKENIZER="${DETOKENIZER:-auto}" +TEXT_COLUMN="${TEXT_COLUMN:-text}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29672}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2000}" +LEARNING_RATE="${LEARNING_RATE:-6e-4}" +MIN_LR="${MIN_LR:-6e-5}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" +OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" +ADAM_BETA1="${ADAM_BETA1:-0.9}" +ADAM_BETA2="${ADAM_BETA2:-0.95}" +ADAM_EPS="${ADAM_EPS:-1e-8}" +OPTIMIZER="${OPTIMIZER:-adamw}" +MUON_MOMENTUM="${MUON_MOMENTUM:-0.95}" +MUON_NS_STEPS="${MUON_NS_STEPS:-5}" +MUON_UPDATE_SCALE="${MUON_UPDATE_SCALE:-1.0}" +EMA_DECAY="${EMA_DECAY:-0.0}" +EMA_START_STEP="${EMA_START_STEP:-0}" +LR_SCHEDULE="${LR_SCHEDULE:-cosine}" +GRAD_CLIP="${GRAD_CLIP:-1.0}" +ADAMW_PARAM_GROUPS="${ADAMW_PARAM_GROUPS:-nanogpt}" +D_MODEL="${D_MODEL:-768}" +COND_DIM="${COND_DIM:-128}" +N_LAYERS="${N_LAYERS:-12}" +N_HEADS="${N_HEADS:-12}" +DIM_FF="${DIM_FF:-3072}" +DROPOUT="${DROPOUT:-0.0}" +MAX_LEN="${MAX_LEN:-1024}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +OWT_CACHED_CHUNKS="${OWT_CACHED_CHUNKS:-0}" +OWT_CHUNK_CACHE_DIR="${OWT_CHUNK_CACHE_DIR:-}" +OWT_CHUNK_CACHE_REBUILD="${OWT_CHUNK_CACHE_REBUILD:-0}" +OWT_CHUNK_CACHE_WRITE_BATCH="${OWT_CHUNK_CACHE_WRITE_BATCH:-4096}" +NUM_WORKERS="${NUM_WORKERS:-4}" +DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-2}" +ALLOW_TF32="${ALLOW_TF32:-1}" +ACTIVATION_CHECKPOINTING="${ACTIVATION_CHECKPOINTING:-0}" +ACTIVATION_CHECKPOINT_INTERVAL="${ACTIVATION_CHECKPOINT_INTERVAL:-1}" +ACTIVATION_CHECKPOINT_SCOPE="${ACTIVATION_CHECKPOINT_SCOPE:-block}" +DDP_STATIC_GRAPH="${DDP_STATIC_GRAPH:-0}" +DDP_GRADIENT_AS_BUCKET_VIEW="${DDP_GRADIENT_AS_BUCKET_VIEW:-1}" +BLOCKING_DATA_TRANSFER="${BLOCKING_DATA_TRANSFER:-0}" +FULL_TRAIN_STATS="${FULL_TRAIN_STATS:-0}" +ROLLOUT_TRAIN_PROB="${ROLLOUT_TRAIN_PROB:-0.0}" +ROLLOUT_TRAIN_STEPS="${ROLLOUT_TRAIN_STEPS:-1}" +ROLLOUT_TRAIN_INFER_STEPS="${ROLLOUT_TRAIN_INFER_STEPS:-64}" +ROLLOUT_TRAIN_TIME_MODE="${ROLLOUT_TRAIN_TIME_MODE:-fixed_steps}" +ROLLOUT_TRAIN_S_DIST="${ROLLOUT_TRAIN_S_DIST:-uniform}" +ROLLOUT_TRAIN_S_MIN_FRAC="${ROLLOUT_TRAIN_S_MIN_FRAC:-0.0}" +ROLLOUT_TRAIN_S_MAX_FRAC="${ROLLOUT_TRAIN_S_MAX_FRAC:-0.125}" +ROLLOUT_TRAIN_S_BETA_ALPHA="${ROLLOUT_TRAIN_S_BETA_ALPHA:-2.0}" +ROLLOUT_TRAIN_S_BETA_BETA="${ROLLOUT_TRAIN_S_BETA_BETA:-6.0}" +ROLLOUT_TRAIN_TEMP="${ROLLOUT_TRAIN_TEMP:-1.45}" +ROLLOUT_TRAIN_MAX_GAMMA="${ROLLOUT_TRAIN_MAX_GAMMA:-1.0}" +ROLLOUT_TRAIN_CORRUPT_ONLY="${ROLLOUT_TRAIN_CORRUPT_ONLY:-1}" +ROLLOUT_TRAIN_SAMPLEWISE="${ROLLOUT_TRAIN_SAMPLEWISE:-0}" +ROLLOUT_TRAIN_SELECTED_ONLY="${ROLLOUT_TRAIN_SELECTED_ONLY:-0}" +ROLLOUT_TRAIN_COMPUTE_ALWAYS="${ROLLOUT_TRAIN_COMPUTE_ALWAYS:-0}" +ROLLOUT_TRAIN_DEBUG_RETURN_BASE="${ROLLOUT_TRAIN_DEBUG_RETURN_BASE:-0}" +ROLLOUT_TRAIN_SYNC_T="${ROLLOUT_TRAIN_SYNC_T:-0}" +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +MASK_MIXTURE_ORIGINAL_PROB="${MASK_MIXTURE_ORIGINAL_PROB:-0.0}" +MASK_MIXTURE_LOWK_PROB="${MASK_MIXTURE_LOWK_PROB:-0.0}" +MASK_MIXTURE_LOWCORRUPT_PROB="${MASK_MIXTURE_LOWCORRUPT_PROB:-0.0}" +MASK_MIXTURE_BLOCK_PROB="${MASK_MIXTURE_BLOCK_PROB:-0.0}" +MASK_MIXTURE_ALL_PROB="${MASK_MIXTURE_ALL_PROB:-0.0}" +MASK_MIXTURE_LOWK_CLEAN_TOKENS="${MASK_MIXTURE_LOWK_CLEAN_TOKENS:-1,2,4,8,16,32,64}" +MASK_MIXTURE_LOWCORRUPT_TOKENS="${MASK_MIXTURE_LOWCORRUPT_TOKENS:-1,2,4,8,16,32,64}" +MASK_MIXTURE_BLOCK_TOKENS="${MASK_MIXTURE_BLOCK_TOKENS:-64,128}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-20000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}" + +if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then + ENABLE_TORCH_COMPILE=0 +fi + +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi +OWT_CACHE_ARGS=() +if [[ "${OWT_CACHED_CHUNKS}" == "1" ]]; then + OWT_CACHE_ARGS+=(--owt_cached_chunks) + if [[ -n "${OWT_CHUNK_CACHE_DIR}" ]]; then + OWT_CACHE_ARGS+=(--owt_chunk_cache_dir "${OWT_CHUNK_CACHE_DIR}") + fi + if [[ "${OWT_CHUNK_CACHE_REBUILD}" == "1" ]]; then + OWT_CACHE_ARGS+=(--owt_chunk_cache_rebuild) + fi + OWT_CACHE_ARGS+=(--owt_chunk_cache_write_batch "${OWT_CHUNK_CACHE_WRITE_BATCH}") +fi +PERF_ARGS=() +if [[ "${ALLOW_TF32}" == "1" ]]; then + PERF_ARGS+=(--allow_tf32) +else + PERF_ARGS+=(--no-allow_tf32) +fi +if [[ "${ACTIVATION_CHECKPOINTING}" == "1" ]]; then + PERF_ARGS+=( + --activation_checkpointing + --activation_checkpoint_interval "${ACTIVATION_CHECKPOINT_INTERVAL}" + --activation_checkpoint_scope "${ACTIVATION_CHECKPOINT_SCOPE}" + ) +fi +if [[ "${DDP_STATIC_GRAPH}" == "1" ]]; then + PERF_ARGS+=(--ddp_static_graph) +fi +if [[ "${DDP_GRADIENT_AS_BUCKET_VIEW}" == "1" ]]; then + PERF_ARGS+=(--ddp_gradient_as_bucket_view) +else + PERF_ARGS+=(--no-ddp_gradient_as_bucket_view) +fi +if [[ "${BLOCKING_DATA_TRANSFER}" == "1" ]]; then + PERF_ARGS+=(--blocking_data_transfer) +fi +if [[ "${FULL_TRAIN_STATS}" == "1" ]]; then + PERF_ARGS+=(--full_train_stats) +fi +ROLLOUT_ARGS=( + --rollout_train_prob "${ROLLOUT_TRAIN_PROB}" + --rollout_train_steps "${ROLLOUT_TRAIN_STEPS}" + --rollout_train_infer_steps "${ROLLOUT_TRAIN_INFER_STEPS}" + --rollout_train_time_mode "${ROLLOUT_TRAIN_TIME_MODE}" + --rollout_train_s_dist "${ROLLOUT_TRAIN_S_DIST}" + --rollout_train_s_min_frac "${ROLLOUT_TRAIN_S_MIN_FRAC}" + --rollout_train_s_max_frac "${ROLLOUT_TRAIN_S_MAX_FRAC}" + --rollout_train_s_beta_alpha "${ROLLOUT_TRAIN_S_BETA_ALPHA}" + --rollout_train_s_beta_beta "${ROLLOUT_TRAIN_S_BETA_BETA}" + --rollout_train_temp "${ROLLOUT_TRAIN_TEMP}" + --rollout_train_max_gamma "${ROLLOUT_TRAIN_MAX_GAMMA}" +) +if [[ "${ROLLOUT_TRAIN_CORRUPT_ONLY}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_corrupt_only) +else + ROLLOUT_ARGS+=(--no-rollout_train_corrupt_only) +fi +if [[ "${ROLLOUT_TRAIN_SAMPLEWISE}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_samplewise) +else + ROLLOUT_ARGS+=(--no-rollout_train_samplewise) +fi +if [[ "${ROLLOUT_TRAIN_SELECTED_ONLY}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_selected_only) +else + ROLLOUT_ARGS+=(--no-rollout_train_selected_only) +fi +if [[ "${ROLLOUT_TRAIN_COMPUTE_ALWAYS}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_compute_always) +else + ROLLOUT_ARGS+=(--no-rollout_train_compute_always) +fi +if [[ "${ROLLOUT_TRAIN_DEBUG_RETURN_BASE}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_debug_return_base) +fi +if [[ "${ROLLOUT_TRAIN_SYNC_T}" == "1" ]]; then + ROLLOUT_ARGS+=(--rollout_train_sync_t) +fi + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" +echo "[launch] method=owt_categorical_fullvocab_${C_TAG}_fullycoupled host=$(hostname) time=$(date -Iseconds)" +echo "[launch] cwd=$(pwd)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" +echo "[launch] data_path=${DATA_PATH}" +echo "[launch] tokenizer=${TOKENIZER_PATH}" +echo "[launch] split=${OPENWEBTEXT_SPLIT} text_column=${TEXT_COLUMN}" +echo "[launch] owt_cached_chunks=${OWT_CACHED_CHUNKS} cache_dir=${OWT_CHUNK_CACHE_DIR}" +echo "[launch] nproc_per_node=${NPROC_PER_NODE} global_batch_size=${GLOBAL_BATCH_SIZE} per_gpu_batch_size=${PER_GPU_BATCH_SIZE}" +echo "[launch] model d_model=${D_MODEL} n_layers=${N_LAYERS} n_heads=${N_HEADS} dim_ff=${DIM_FF} dropout=${DROPOUT}" +echo "[launch] optimizer=${OPTIMIZER} lr=${LEARNING_RATE} wd=${WEIGHT_DECAY} output_wd=${OUTPUT_WEIGHT_DECAY} ema=${EMA_DECAY}" +echo "[launch] rollout_train prob=${ROLLOUT_TRAIN_PROB} mode=${ROLLOUT_TRAIN_TIME_MODE} steps=${ROLLOUT_TRAIN_STEPS} infer_steps=${ROLLOUT_TRAIN_INFER_STEPS} s_dist=${ROLLOUT_TRAIN_S_DIST} s_frac=${ROLLOUT_TRAIN_S_MIN_FRAC}->${ROLLOUT_TRAIN_S_MAX_FRAC} temp=${ROLLOUT_TRAIN_TEMP} max_gamma=${ROLLOUT_TRAIN_MAX_GAMMA} corrupt_only=${ROLLOUT_TRAIN_CORRUPT_ONLY} samplewise=${ROLLOUT_TRAIN_SAMPLEWISE} selected_only=${ROLLOUT_TRAIN_SELECTED_ONLY} compute_always=${ROLLOUT_TRAIN_COMPUTE_ALWAYS} sync_t=${ROLLOUT_TRAIN_SYNC_T}" +echo "[launch] mask min=${MIN_MASK_RATIO} max=${MAX_MASK_RATIO} mixture_all=${MASK_MIXTURE_ALL_PROB} mixture_original=${MASK_MIXTURE_ORIGINAL_PROB} mixture_lowk=${MASK_MIXTURE_LOWK_PROB} mixture_lowcorrupt=${MASK_MIXTURE_LOWCORRUPT_PROB} mixture_block=${MASK_MIXTURE_BLOCK_PROB}" +echo "[launch] perf allow_tf32=${ALLOW_TF32} activation_checkpointing=${ACTIVATION_CHECKPOINTING} checkpoint_interval=${ACTIVATION_CHECKPOINT_INTERVAL} checkpoint_scope=${ACTIVATION_CHECKPOINT_SCOPE} prefetch=${DATALOADER_PREFETCH_FACTOR}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --detokenizer "${DETOKENIZER}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + "${OWT_CACHE_ARGS[@]}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr "${LEARNING_RATE}" \ + --weight_decay "${WEIGHT_DECAY}" \ + --output_weight_decay "${OUTPUT_WEIGHT_DECAY}" \ + --adam_beta1 "${ADAM_BETA1}" \ + --adam_beta2 "${ADAM_BETA2}" \ + --adam_eps "${ADAM_EPS}" \ + --optimizer "${OPTIMIZER}" \ + --muon_momentum "${MUON_MOMENTUM}" \ + --muon_ns_steps "${MUON_NS_STEPS}" \ + --muon_update_scale "${MUON_UPDATE_SCALE}" \ + --ema_decay "${EMA_DECAY}" \ + --ema_start_step "${EMA_START_STEP}" \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule "${LR_SCHEDULE}" \ + --min_lr "${MIN_LR}" \ + --grad_clip "${GRAD_CLIP}" \ + --adamw_param_groups "${ADAMW_PARAM_GROUPS}" \ + --seed 123 \ + --d_model "${D_MODEL}" \ + --cond_dim "${COND_DIM}" \ + --n_layers "${N_LAYERS}" \ + --n_heads "${N_HEADS}" \ + --dim_ff "${DIM_FF}" \ + --dropout "${DROPOUT}" \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + "${ROLLOUT_ARGS[@]}" \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --mask_mixture_original_prob "${MASK_MIXTURE_ORIGINAL_PROB}" \ + --mask_mixture_lowk_prob "${MASK_MIXTURE_LOWK_PROB}" \ + --mask_mixture_lowcorrupt_prob "${MASK_MIXTURE_LOWCORRUPT_PROB}" \ + --mask_mixture_block_prob "${MASK_MIXTURE_BLOCK_PROB}" \ + --mask_mixture_all_prob "${MASK_MIXTURE_ALL_PROB}" \ + --mask_mixture_lowk_clean_tokens "${MASK_MIXTURE_LOWK_CLEAN_TOKENS}" \ + --mask_mixture_lowcorrupt_tokens "${MASK_MIXTURE_LOWCORRUPT_TOKENS}" \ + --mask_mixture_block_tokens "${MASK_MIXTURE_BLOCK_TOKENS}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max "${C_MAX}" \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 1024 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + "${PERF_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_elfaligned_t5_logitnormal_tokenized_8gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_elfaligned_t5_logitnormal_tokenized_8gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..aab82722f0931840a271948174c512cb62b0840d --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_elfaligned_t5_logitnormal_tokenized_8gpu.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-32117}" + +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +EPOCHS="${EPOCHS:-5}" +NUM_WORKERS="${NUM_WORKERS:-8}" +DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-4}" +LOG_EVERY="${LOG_EVERY:-100}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ALLOW_TF32="${ALLOW_TF32:-1}" +DRY_RUN="${DRY_RUN:-0}" +TOTAL_STEPS="${TOTAL_STEPS:-1}" + +LR="${LR:-0.002}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" +OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" +ADAM_BETA1="${ADAM_BETA1:-0.9}" +ADAM_BETA2="${ADAM_BETA2:-0.999}" +ADAM_EPS="${ADAM_EPS:-1e-8}" +MUON_MOMENTUM="${MUON_MOMENTUM:-0.95}" +MUON_NS_STEPS="${MUON_NS_STEPS:-5}" +MUON_UPDATE_SCALE="${MUON_UPDATE_SCALE:-1.0}" +GRAD_CLIP="${GRAD_CLIP:-1.0}" +EMA_DECAY="${EMA_DECAY:-0.9999}" +EMA_START_STEP="${EMA_START_STEP:-0}" +T_LOGIT_MEAN="${T_LOGIT_MEAN:--1.5}" +T_LOGIT_STD="${T_LOGIT_STD:-0.8}" +T_GUMBEL_LOC="${T_GUMBEL_LOC:-2.2}" +T_GUMBEL_SCALE="${T_GUMBEL_SCALE:-0.8}" +T_SAMPLING_MODE="${T_SAMPLING_MODE:-logit_normal}" +LOSS_T_WEIGHT_MODE="${LOSS_T_WEIGHT_MODE:-none}" +LOSS_T_MIN_WEIGHT="${LOSS_T_MIN_WEIGHT:-0.0}" +TARGET_LOSS="${TARGET_LOSS:-hard_ce}" +LINEAR_SOFT_TARGET_POWER="${LINEAR_SOFT_TARGET_POWER:-1.0}" +LINEAR_SOFT_TARGET_MIN_CONF="${LINEAR_SOFT_TARGET_MIN_CONF:-0.0}" +LINEAR_SOFT_TARGET_MAX_CONF="${LINEAR_SOFT_TARGET_MAX_CONF:-1.0}" +OUTPUT_INIT_STD="${OUTPUT_INIT_STD:-0.0}" +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +MASK_MIXTURE_ORIGINAL_PROB="${MASK_MIXTURE_ORIGINAL_PROB:-0.0}" +MASK_MIXTURE_LOWK_PROB="${MASK_MIXTURE_LOWK_PROB:-0.0}" +MASK_MIXTURE_LOWCORRUPT_PROB="${MASK_MIXTURE_LOWCORRUPT_PROB:-0.0}" +MASK_MIXTURE_BLOCK_PROB="${MASK_MIXTURE_BLOCK_PROB:-0.0}" +MASK_MIXTURE_ALL_PROB="${MASK_MIXTURE_ALL_PROB:-0.0}" +MASK_MIXTURE_LOWK_CLEAN_TOKENS="${MASK_MIXTURE_LOWK_CLEAN_TOKENS:-1,2,4,8,16,32,64}" +MASK_MIXTURE_LOWCORRUPT_TOKENS="${MASK_MIXTURE_LOWCORRUPT_TOKENS:-1,2,4,8,16,32,64}" +MASK_MIXTURE_BLOCK_TOKENS="${MASK_MIXTURE_BLOCK_TOKENS:-64,128}" +ROLLOUT_TRAIN_PROB="${ROLLOUT_TRAIN_PROB:-0.0}" +ROLLOUT_TRAIN_STEPS="${ROLLOUT_TRAIN_STEPS:-1}" +ROLLOUT_TRAIN_STEPS_MIN="${ROLLOUT_TRAIN_STEPS_MIN:--1}" +ROLLOUT_TRAIN_INFER_STEPS="${ROLLOUT_TRAIN_INFER_STEPS:-64}" +ROLLOUT_TRAIN_TIME_MODE="${ROLLOUT_TRAIN_TIME_MODE:-fixed_steps}" +ROLLOUT_TRAIN_S_DIST="${ROLLOUT_TRAIN_S_DIST:-uniform}" +ROLLOUT_TRAIN_S_MIN_FRAC="${ROLLOUT_TRAIN_S_MIN_FRAC:-0.0}" +ROLLOUT_TRAIN_S_MAX_FRAC="${ROLLOUT_TRAIN_S_MAX_FRAC:-0.125}" +ROLLOUT_TRAIN_S_BETA_ALPHA="${ROLLOUT_TRAIN_S_BETA_ALPHA:-2.0}" +ROLLOUT_TRAIN_S_BETA_BETA="${ROLLOUT_TRAIN_S_BETA_BETA:-6.0}" +ROLLOUT_TRAIN_TEMP="${ROLLOUT_TRAIN_TEMP:-1.0}" +ROLLOUT_TRAIN_MAX_GAMMA="${ROLLOUT_TRAIN_MAX_GAMMA:-1.0}" +ROLLOUT_TRAIN_RULE="${ROLLOUT_TRAIN_RULE:-flowmap}" +ROLLOUT_TRAIN_CORRUPT_ONLY="${ROLLOUT_TRAIN_CORRUPT_ONLY:-1}" +ROLLOUT_TRAIN_SAMPLEWISE="${ROLLOUT_TRAIN_SAMPLEWISE:-0}" +ROLLOUT_TRAIN_SELECTED_ONLY="${ROLLOUT_TRAIN_SELECTED_ONLY:-0}" +ROLLOUT_TRAIN_COMPUTE_ALWAYS="${ROLLOUT_TRAIN_COMPUTE_ALWAYS:-0}" +ROLLOUT_TRAIN_SYNC_T="${ROLLOUT_TRAIN_SYNC_T:-0}" + +sanitize_label() { + printf "%s" "$1" | sed -e 's/-/m/g' -e 's/\./p/g' +} + +T_LOGIT_MEAN_LABEL="$(sanitize_label "${T_LOGIT_MEAN}")" +T_LOGIT_STD_LABEL="$(sanitize_label "${T_LOGIT_STD}")" +LOSS_T_MIN_WEIGHT_LABEL="$(sanitize_label "${LOSS_T_MIN_WEIGHT}")" + +RUN_NAME="${RUN_NAME:-lta_owt_elfofficial_t5_len1024_elfaligned_dditelf_muon_logitnormal_${T_LOGIT_MEAN_LABEL}_s${T_LOGIT_STD_LABEL}_${LOSS_T_WEIGHT_MODE}_floor${LOSS_T_MIN_WEIGHT_LABEL}_gbs512_8gpu_5epoch_$(date +%Y%m%d_%H%M%S)}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_DIR="${LOG_DIR:-logs/elfaligned_t5tokenized_8gpu}" +LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.log}" + +NUM_EXAMPLES=$(python - <&2 + echo "Use a new RUN_NAME/SAVE_DIR or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p "${LOG_DIR}" "${SAVE_DIR}" + +TF32_FLAG="--allow_tf32" +TF32_LABEL="true" +if [[ "${ALLOW_TF32}" == "0" || "${ALLOW_TF32}" == "false" || "${ALLOW_TF32}" == "False" ]]; then + TF32_FLAG="--no-allow_tf32" + TF32_LABEL="false" +fi + +bool_enabled() { + case "$1" in + 1|true|True|TRUE|yes|Yes|YES|on|On|ON) return 0 ;; + *) return 1 ;; + esac +} + +ROLLOUT_BOOL_FLAGS=() +if bool_enabled "${ROLLOUT_TRAIN_CORRUPT_ONLY}"; then + ROLLOUT_BOOL_FLAGS+=(--rollout_train_corrupt_only) +else + ROLLOUT_BOOL_FLAGS+=(--no-rollout_train_corrupt_only) +fi +if bool_enabled "${ROLLOUT_TRAIN_SAMPLEWISE}"; then + ROLLOUT_BOOL_FLAGS+=(--rollout_train_samplewise) +else + ROLLOUT_BOOL_FLAGS+=(--no-rollout_train_samplewise) +fi +if bool_enabled "${ROLLOUT_TRAIN_SELECTED_ONLY}"; then + ROLLOUT_BOOL_FLAGS+=(--rollout_train_selected_only) +else + ROLLOUT_BOOL_FLAGS+=(--no-rollout_train_selected_only) +fi +if bool_enabled "${ROLLOUT_TRAIN_COMPUTE_ALWAYS}"; then + ROLLOUT_BOOL_FLAGS+=(--rollout_train_compute_always) +else + ROLLOUT_BOOL_FLAGS+=(--no-rollout_train_compute_always) +fi +if bool_enabled "${ROLLOUT_TRAIN_SYNC_T}"; then + ROLLOUT_BOOL_FLAGS+=(--rollout_train_sync_t) +else + ROLLOUT_BOOL_FLAGS+=(--no-rollout_train_sync_t) +fi + +echo "[launch] method=owt_elfaligned_t5tokenized_dditelf host=$(hostname) time=$(date -Iseconds)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" +echo "[launch] data_path=${DATA_PATH}" +echo "[launch] tokenizer=${TOKENIZER_PATH}" +echo "[launch] examples=${NUM_EXAMPLES} epochs=${EPOCHS} world_size=${WORLD_SIZE} grad_accum=${GRAD_ACCUM} loader_batches_per_rank=${LOADER_BATCHES_PER_RANK} steps_per_epoch=${STEPS_PER_EPOCH} save_every=${SAVE_EVERY}" +echo "[launch] optimizer=muon_impl=optax grouping=hidden_2d lr=${LR} hidden_wd=0 adam_fallback_wd=${WEIGHT_DECAY} momentum=${MUON_MOMENTUM} ns=${MUON_NS_STEPS} nesterov=true width_scale=true adam_fallback_b2=${ADAM_BETA2} ema=${EMA_DECAY}" +echo "[launch] model=ddit_elf rmsnorm qk_norm=true swiglu no_adaln output_bias=false output_init_std=${OUTPUT_INIT_STD} time_tokens=4 mode_tokens=0" +echo "[launch] data=tokenized_hf pad=pad add_special_tokens=false t5-small fp32=true bf16=false tf32=${TF32_LABEL}" +echo "[launch] t_sampling=${T_SAMPLING_MODE} mean=${T_LOGIT_MEAN} std=${T_LOGIT_STD} gumbel=${T_GUMBEL_LOC},${T_GUMBEL_SCALE} target_loss=${TARGET_LOSS} linear_soft_power=${LINEAR_SOFT_TARGET_POWER} linear_soft_conf=${LINEAR_SOFT_TARGET_MIN_CONF}->${LINEAR_SOFT_TARGET_MAX_CONF} loss_t_weight=${LOSS_T_WEIGHT_MODE} loss_t_min_weight=${LOSS_T_MIN_WEIGHT} warmup_epochs=0.5" +echo "[launch] mask=min${MIN_MASK_RATIO}->max${MAX_MASK_RATIO} mixture original=${MASK_MIXTURE_ORIGINAL_PROB} lowk=${MASK_MIXTURE_LOWK_PROB} lowcorrupt=${MASK_MIXTURE_LOWCORRUPT_PROB} block=${MASK_MIXTURE_BLOCK_PROB} all=${MASK_MIXTURE_ALL_PROB}" +echo "[launch] rollout_train prob=${ROLLOUT_TRAIN_PROB} mode=${ROLLOUT_TRAIN_TIME_MODE} steps=${ROLLOUT_TRAIN_STEPS} steps_min=${ROLLOUT_TRAIN_STEPS_MIN} infer_steps=${ROLLOUT_TRAIN_INFER_STEPS} s=${ROLLOUT_TRAIN_S_DIST}:${ROLLOUT_TRAIN_S_MIN_FRAC}->${ROLLOUT_TRAIN_S_MAX_FRAC} temp=${ROLLOUT_TRAIN_TEMP} samplewise=${ROLLOUT_TRAIN_SAMPLEWISE} selected_only=${ROLLOUT_TRAIN_SELECTED_ONLY} sync_t=${ROLLOUT_TRAIN_SYNC_T}" + +if [[ "${DRY_RUN}" == "1" || "${DRY_RUN}" == "true" || "${DRY_RUN}" == "True" ]]; then + echo "[launch] DRY_RUN=1, validated data path and step schedule; skipping torchrun." + exit 0 +fi + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + --tokenized_hf \ + --tokenized_pad_token pad \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --max_len 1024 \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR}" \ + --epochs "${EPOCHS}" \ + --total_steps "${TOTAL_STEPS}" \ + --warmup_epochs 0.5 \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --optimizer muon \ + --muon_impl optax \ + --lr "${LR}" \ + --lr_schedule constant_warmup \ + --min_lr 0 \ + --weight_decay "${WEIGHT_DECAY}" \ + --output_weight_decay "${OUTPUT_WEIGHT_DECAY}" \ + --adam_beta1 "${ADAM_BETA1}" \ + --adam_beta2 "${ADAM_BETA2}" \ + --adam_eps "${ADAM_EPS}" \ + --muon_momentum "${MUON_MOMENTUM}" \ + --muon_ns_steps "${MUON_NS_STEPS}" \ + --muon_update_scale "${MUON_UPDATE_SCALE}" \ + --muon_nesterov \ + --muon_width_scale \ + --ema_decay "${EMA_DECAY}" \ + --ema_start_step "${EMA_START_STEP}" \ + --grad_clip "${GRAD_CLIP}" \ + --seed 42 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.0 \ + --no-output_bias \ + --output_init_std "${OUTPUT_INIT_STD}" \ + --norm_type rmsnorm \ + --model_type ddit_elf \ + --elf_num_time_tokens 4 \ + --elf_num_model_mode_tokens 0 \ + --qk_norm \ + --state_format prob \ + --bridge dirichlet \ + --target_loss "${TARGET_LOSS}" \ + --linear_soft_target_power "${LINEAR_SOFT_TARGET_POWER}" \ + --linear_soft_target_min_conf "${LINEAR_SOFT_TARGET_MIN_CONF}" \ + --linear_soft_target_max_conf "${LINEAR_SOFT_TARGET_MAX_CONF}" \ + --loss_t_weight_mode "${LOSS_T_WEIGHT_MODE}" \ + --loss_t_min_weight "${LOSS_T_MIN_WEIGHT}" \ + --rollout_train_prob "${ROLLOUT_TRAIN_PROB}" \ + --rollout_train_steps "${ROLLOUT_TRAIN_STEPS}" \ + --rollout_train_steps_min "${ROLLOUT_TRAIN_STEPS_MIN}" \ + --rollout_train_infer_steps "${ROLLOUT_TRAIN_INFER_STEPS}" \ + --rollout_train_time_mode "${ROLLOUT_TRAIN_TIME_MODE}" \ + --rollout_train_s_dist "${ROLLOUT_TRAIN_S_DIST}" \ + --rollout_train_s_min_frac "${ROLLOUT_TRAIN_S_MIN_FRAC}" \ + --rollout_train_s_max_frac "${ROLLOUT_TRAIN_S_MAX_FRAC}" \ + --rollout_train_s_beta_alpha "${ROLLOUT_TRAIN_S_BETA_ALPHA}" \ + --rollout_train_s_beta_beta "${ROLLOUT_TRAIN_S_BETA_BETA}" \ + --rollout_train_temp "${ROLLOUT_TRAIN_TEMP}" \ + --rollout_train_max_gamma "${ROLLOUT_TRAIN_MAX_GAMMA}" \ + --rollout_train_rule "${ROLLOUT_TRAIN_RULE}" \ + "${ROLLOUT_BOOL_FLAGS[@]}" \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --t_sampling_mode "${T_SAMPLING_MODE}" \ + --t_sampling_logit_mean "${T_LOGIT_MEAN}" \ + --t_sampling_logit_std "${T_LOGIT_STD}" \ + --t_sampling_gumbel_loc "${T_GUMBEL_LOC}" \ + --t_sampling_gumbel_scale "${T_GUMBEL_SCALE}" \ + --t_sampling_eps 1e-4 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --mask_mixture_original_prob "${MASK_MIXTURE_ORIGINAL_PROB}" \ + --mask_mixture_lowk_prob "${MASK_MIXTURE_LOWK_PROB}" \ + --mask_mixture_lowcorrupt_prob "${MASK_MIXTURE_LOWCORRUPT_PROB}" \ + --mask_mixture_block_prob "${MASK_MIXTURE_BLOCK_PROB}" \ + --mask_mixture_all_prob "${MASK_MIXTURE_ALL_PROB}" \ + --mask_mixture_lowk_clean_tokens "${MASK_MIXTURE_LOWK_CLEAN_TOKENS}" \ + --mask_mixture_lowcorrupt_tokens "${MASK_MIXTURE_LOWCORRUPT_TOKENS}" \ + --mask_mixture_block_tokens "${MASK_MIXTURE_BLOCK_TOKENS}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 1024 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${TF32_FLAG}" \ + --activation_checkpointing \ + --activation_checkpoint_scope mlp \ + --ddp_gradient_as_bucket_view \ + 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_fullycoupled_wd0p1_fp32_8gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_fullycoupled_wd0p1_fp32_8gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..b93ac45391e6d0db9d7c842c9df155b6faafb835 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_fullycoupled_wd0p1_fp32_8gpu.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +# Fully-coupled OWT baseline with FP32 params/activations and nanoGPT-style AdamW decay: +# decay: matrix / embedding params (p.dim() >= 2) +# no decay: bias / norm / 1D params +# This intentionally does NOT pass --bf16. TF32 is enabled by default so H200 uses +# Tensor Cores; set ALLOW_TF32=0 for strict FP32 debugging. + +RUN_NAME="${RUN_NAME:-lta_owt_gpt2cached_len1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_nanogpt_fp32_ddit768x12_gbs512_8gpu_1m_$(date +%Y%m%d_%H%M%S)}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_DIR="${LOG_DIR:-logs/fullycoupled_wd0p1_fp32_8gpu}" +LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.log}" + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +OWT_CACHE="${OWT_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-31997}" + +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2000}" +NUM_WORKERS="${NUM_WORKERS:-8}" +DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-4}" +LOG_EVERY="${LOG_EVERY:-50}" +SAVE_EVERY="${SAVE_EVERY:-50000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ALLOW_TF32="${ALLOW_TF32:-1}" + +LR="${LR:-6e-4}" +MIN_LR="${MIN_LR:-6e-5}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" +OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" +ADAM_BETA1="${ADAM_BETA1:-0.9}" +ADAM_BETA2="${ADAM_BETA2:-0.95}" +ADAM_EPS="${ADAM_EPS:-1e-8}" +GRAD_CLIP="${GRAD_CLIP:-1.0}" +EMA_DECAY="${EMA_DECAY:-0.0}" +EMA_START_STEP="${EMA_START_STEP:-0}" +LOSS_T_WEIGHT_MODE="${LOSS_T_WEIGHT_MODE:-none}" +LOSS_T_MIN_WEIGHT="${LOSS_T_MIN_WEIGHT:-0.0}" +LOSS_T_DROP_BELOW="${LOSS_T_DROP_BELOW:-0.2}" +T_SAMPLING_MODE="${T_SAMPLING_MODE:-uniform}" +T_SAMPLING_POWER="${T_SAMPLING_POWER:-1.0}" +T_SAMPLING_EPS="${T_SAMPLING_EPS:-1e-4}" + +if [[ -f "${SAVE_DIR}/args.json" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p "${LOG_DIR}" "${SAVE_DIR}" + +TF32_FLAG="--allow_tf32" +TF32_LABEL="true" +if [[ "${ALLOW_TF32}" == "0" || "${ALLOW_TF32}" == "false" || "${ALLOW_TF32}" == "False" ]]; then + TF32_FLAG="--no-allow_tf32" + TF32_LABEL="false" +fi + +echo "[launch] method=owt_fullycoupled_adamw_wd0p1_nanogpt_fp32 host=$(hostname) time=$(date -Iseconds)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" +echo "[launch] data_path=${DATA_PATH}" +echo "[launch] owt_cache=${OWT_CACHE}" +echo "[launch] optimizer=adamw lr=${LR} min_lr=${MIN_LR} wd=${WEIGHT_DECAY} output_wd=${OUTPUT_WEIGHT_DECAY} param_groups=nanogpt ema=${EMA_DECAY}" +echo "[launch] fp32=true bf16=false tf32=${TF32_LABEL} norm_type=rmsnorm output_bias=false batch=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE}" +echo "[launch] loss_t_weight_mode=${LOSS_T_WEIGHT_MODE} loss_t_min_weight=${LOSS_T_MIN_WEIGHT} loss_t_drop_below=${LOSS_T_DROP_BELOW}" +echo "[launch] t_sampling_mode=${T_SAMPLING_MODE} t_sampling_power=${T_SAMPLING_POWER} t_sampling_eps=${T_SAMPLING_EPS}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + --openwebtext_split train_minus_100k \ + --text_column text \ + --detokenizer auto \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode stream \ + --owt_cached_chunks \ + --owt_chunk_cache_dir "${OWT_CACHE}" \ + --max_len 1024 \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR}" \ + --total_steps "${TOTAL_STEPS}" \ + --warmup_steps "${WARMUP_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --optimizer adamw \ + --lr "${LR}" \ + --lr_schedule cosine \ + --min_lr "${MIN_LR}" \ + --weight_decay "${WEIGHT_DECAY}" \ + --output_weight_decay "${OUTPUT_WEIGHT_DECAY}" \ + --adamw_param_groups nanogpt \ + --adam_beta1 "${ADAM_BETA1}" \ + --adam_beta2 "${ADAM_BETA2}" \ + --adam_eps "${ADAM_EPS}" \ + --ema_decay "${EMA_DECAY}" \ + --ema_start_step "${EMA_START_STEP}" \ + --grad_clip "${GRAD_CLIP}" \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.0 \ + --no-output_bias \ + --norm_type rmsnorm \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --loss_t_weight_mode "${LOSS_T_WEIGHT_MODE}" \ + --loss_t_min_weight "${LOSS_T_MIN_WEIGHT}" \ + --loss_t_drop_below "${LOSS_T_DROP_BELOW}" \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --t_sampling_mode "${T_SAMPLING_MODE}" \ + --t_sampling_power "${T_SAMPLING_POWER}" \ + --t_sampling_eps "${T_SAMPLING_EPS}" \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 1024 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${TF32_FLAG}" \ + --ddp_gradient_as_bucket_view \ + 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_selfcond_p05_8gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_selfcond_p05_8gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..cf9ac487f5a28ad952037a6a988a2357d1a0ad9b --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_selfcond_p05_8gpu.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" +export TOKENIZERS_PARALLELISM=false + +STAMP="$(date +%Y%m%d_%H%M%S)" +export RUN_NAME="${RUN_NAME:-lta_owt_gpt2cached_len1024_selfcond_p05_rollout1_autocastfix_c1024_ddit768x12_muon_ema_gbs512_8gpu_1m_${STAMP}}" +export SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +export LOG_FILE="${LOG_FILE:-logs/selfcond_8gpu/${RUN_NAME}.log}" +mkdir -p "$(dirname "${LOG_FILE}")" + +export NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +export MASTER_PORT="${MASTER_PORT:-31995}" +export PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +export TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +export WARMUP_STEPS="${WARMUP_STEPS:-2000}" +export MAX_LEN="${MAX_LEN:-1024}" +export NUM_WORKERS="${NUM_WORKERS:-8}" +export DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-4}" +export LOG_EVERY="${LOG_EVERY:-50}" +export SAVE_EVERY="${SAVE_EVERY:-1000}" +export LATEST_EVERY="${LATEST_EVERY:-1000}" +export EVAL_EVERY="${EVAL_EVERY:-0}" + +export OWT_CACHED_CHUNKS=1 +export OWT_CHUNK_CACHE_REBUILD=0 +export OWT_CHUNK_CACHE_DIR="${OWT_CHUNK_CACHE_DIR:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k}" + +export OPTIMIZER="${OPTIMIZER:-muon}" +export LEARNING_RATE="${LEARNING_RATE:-0.002}" +export LR_SCHEDULE="${LR_SCHEDULE:-constant_warmup}" +export MIN_LR="${MIN_LR:-0}" +export WEIGHT_DECAY="${WEIGHT_DECAY:-0.0}" +export ADAM_BETA1="${ADAM_BETA1:-0.9}" +export ADAM_BETA2="${ADAM_BETA2:-0.95}" +export ADAM_EPS="${ADAM_EPS:-1e-8}" +export MUON_MOMENTUM="${MUON_MOMENTUM:-0.95}" +export MUON_NS_STEPS="${MUON_NS_STEPS:-5}" +export MUON_UPDATE_SCALE="${MUON_UPDATE_SCALE:-1.0}" +export EMA_DECAY="${EMA_DECAY:-0.9999}" +export EMA_START_STEP="${EMA_START_STEP:-0}" + +export ALLOW_TF32="${ALLOW_TF32:-1}" +export ACTIVATION_CHECKPOINTING="${ACTIVATION_CHECKPOINTING:-0}" +export ACTIVATION_CHECKPOINT_INTERVAL="${ACTIVATION_CHECKPOINT_INTERVAL:-2}" +export DDP_STATIC_GRAPH="${DDP_STATIC_GRAPH:-0}" +export DDP_GRADIENT_AS_BUCKET_VIEW="${DDP_GRADIENT_AS_BUCKET_VIEW:-1}" +export BLOCKING_DATA_TRANSFER="${BLOCKING_DATA_TRANSFER:-0}" +export FULL_TRAIN_STATS="${FULL_TRAIN_STATS:-0}" +export ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +export AUTO_RESUME="${AUTO_RESUME:-1}" +export ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-1}" + +# Self-conditioning / rollout-state training. +# The no-grad rollout is generated outside the supervised autocast context; this +# avoids AMP cast-cache reusing no-grad weights and silently zeroing weight grads. +export ROLLOUT_TRAIN_PROB="${ROLLOUT_TRAIN_PROB:-0.5}" +export ROLLOUT_TRAIN_STEPS="${ROLLOUT_TRAIN_STEPS:-1}" +export ROLLOUT_TRAIN_INFER_STEPS="${ROLLOUT_TRAIN_INFER_STEPS:-64}" +export ROLLOUT_TRAIN_TEMP="${ROLLOUT_TRAIN_TEMP:-1.45}" +export ROLLOUT_TRAIN_MAX_GAMMA="${ROLLOUT_TRAIN_MAX_GAMMA:-1.0}" +export ROLLOUT_TRAIN_CORRUPT_ONLY="${ROLLOUT_TRAIN_CORRUPT_ONLY:-1}" +export ROLLOUT_TRAIN_SAMPLEWISE="${ROLLOUT_TRAIN_SAMPLEWISE:-1}" +export ROLLOUT_TRAIN_COMPUTE_ALWAYS="${ROLLOUT_TRAIN_COMPUTE_ALWAYS:-0}" +export ROLLOUT_TRAIN_DEBUG_RETURN_BASE="${ROLLOUT_TRAIN_DEBUG_RETURN_BASE:-0}" + +if [[ "${AUTO_RESUME}" == "1" && -z "${RESUME_PATH:-}" && -f "${SAVE_DIR}/latest.pt" ]]; then + export RESUME_PATH="${SAVE_DIR}/latest.pt" + echo "[selfcond_8gpu] auto-resume from ${RESUME_PATH}" +fi + +bash scripts/launch_lta_owt_c1024_fullycoupled_8gpu_len1024_gpt2_cached_chunks_1m.sh diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_t5_len128_sde_rollin_lognsr_8gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_t5_len128_sde_rollin_lognsr_8gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..61b3bb351239f9ef9a9b413ca11db0bbedeb1919 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_t5_len128_sde_rollin_lognsr_8gpu.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +export MASTER_PORT="${MASTER_PORT:-39282}" +export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" +export PYTHONUNBUFFERED=1 +export TOKENIZERS_PARALLELISM=false +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json}" + +MAX_LEN="${MAX_LEN:-128}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +NUM_WORKERS="${NUM_WORKERS:-8}" +DATALOADER_PREFETCH_FACTOR="${DATALOADER_PREFETCH_FACTOR:-4}" + +LR="${LR:-0.002}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" +OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" +OUTPUT_INIT_STD="${OUTPUT_INIT_STD:-0.0}" +EMA_DECAY="${EMA_DECAY:-0.9999}" +EMA_START_STEP="${EMA_START_STEP:-0}" + +T_SAMPLING_MODE="${T_SAMPLING_MODE:-lognsr_gumbel}" +T_GUMBEL_LOC="${T_GUMBEL_LOC:-2.2}" +T_GUMBEL_SCALE="${T_GUMBEL_SCALE:-0.8}" +T_LOGIT_MEAN="${T_LOGIT_MEAN:--1.5}" +T_LOGIT_STD="${T_LOGIT_STD:-0.8}" + +TARGET_LOSS="${TARGET_LOSS:-hard_ce}" +LOSS_T_WEIGHT_MODE="${LOSS_T_WEIGHT_MODE:-none}" +LOSS_T_MIN_WEIGHT="${LOSS_T_MIN_WEIGHT:-0.0}" + +ROLLOUT_TRAIN_PROB="${ROLLOUT_TRAIN_PROB:-0.50}" +ROLLOUT_TRAIN_RULE="${ROLLOUT_TRAIN_RULE:-dirichlet_resample}" +ROLLOUT_TRAIN_TIME_MODE="${ROLLOUT_TRAIN_TIME_MODE:-sampled_path}" +ROLLOUT_TRAIN_STEPS="${ROLLOUT_TRAIN_STEPS:-3}" +ROLLOUT_TRAIN_STEPS_MIN="${ROLLOUT_TRAIN_STEPS_MIN:-0}" +ROLLOUT_TRAIN_INFER_STEPS="${ROLLOUT_TRAIN_INFER_STEPS:-1}" +ROLLOUT_TRAIN_S_DIST="${ROLLOUT_TRAIN_S_DIST:-uniform}" +ROLLOUT_TRAIN_S_MIN_FRAC="${ROLLOUT_TRAIN_S_MIN_FRAC:-0.0}" +ROLLOUT_TRAIN_S_MAX_FRAC="${ROLLOUT_TRAIN_S_MAX_FRAC:-0.25}" +ROLLOUT_TRAIN_TEMP="${ROLLOUT_TRAIN_TEMP:-1.0}" +ROLLOUT_TRAIN_MAX_GAMMA="${ROLLOUT_TRAIN_MAX_GAMMA:-1.0}" + +MIN_MASK_RATIO="${MIN_MASK_RATIO:-1.0}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +MASK_MIXTURE_ALL_PROB="${MASK_MIXTURE_ALL_PROB:-1.0}" + +RUN_NAME="${RUN_NAME:-lta_owt_t5_len${MAX_LEN}_sde_rollin_p50_k0_3_lognsr_gumbel_mask1_gbs${GLOBAL_BATCH_SIZE}_${NPROC_PER_NODE}gpu_$(date +%Y%m%d_%H%M%S)}" +SAVE_DIR="runs/${RUN_NAME}" +LOG_DIR="logs/owt_len128_sde_rollin_lognsr" +LOG_FILE="${LOG_DIR}/${RUN_NAME}.log" +mkdir -p "${SAVE_DIR}" "${LOG_DIR}" +RESUME_PATH="${RESUME_PATH:-}" +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS=(--resume_path "${RESUME_PATH}") +fi + +WORLD_SIZE="${NPROC_PER_NODE}" +GRAD_ACCUM=$(( (GLOBAL_BATCH_SIZE + PER_GPU_BATCH_SIZE * WORLD_SIZE - 1) / (PER_GPU_BATCH_SIZE * WORLD_SIZE) )) + +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] data_path=${DATA_PATH} tokenizer=${TOKENIZER_PATH} max_len=${MAX_LEN}" +echo "[launch] gbs=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE} world=${WORLD_SIZE} grad_accum=${GRAD_ACCUM} total_steps=${TOTAL_STEPS} save_every=${SAVE_EVERY}" +echo "[launch] t_sampling=${T_SAMPLING_MODE} gumbel=${T_GUMBEL_LOC},${T_GUMBEL_SCALE} rollout_rule=${ROLLOUT_TRAIN_RULE} rollout_prob=${ROLLOUT_TRAIN_PROB} rollout_steps=${ROLLOUT_TRAIN_STEPS_MIN}..${ROLLOUT_TRAIN_STEPS}" +echo "[launch] resume_path=${RESUME_PATH:-}" + +if [[ "${DRY_RUN:-0}" == "1" ]]; then + echo "[launch] DRY_RUN=1 ok" + exit 0 +fi + +torchrun \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + "${RESUME_ARGS[@]}" \ + --data_path "${DATA_PATH}" \ + --tokenized_hf \ + --tokenized_pad_token pad \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR}" \ + --epochs 0 \ + --total_steps "${TOTAL_STEPS}" \ + --warmup_steps 1 \ + --warmup_epochs 0.5 \ + --log_every "${LOG_EVERY}" \ + --eval_every 0 \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --optimizer muon \ + --muon_impl optax \ + --lr "${LR}" \ + --lr_schedule constant_warmup \ + --min_lr 0.0 \ + --weight_decay "${WEIGHT_DECAY}" \ + --output_weight_decay "${OUTPUT_WEIGHT_DECAY}" \ + --adamw_param_groups nanogpt \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --ema_decay "${EMA_DECAY}" \ + --ema_start_step "${EMA_START_STEP}" \ + --grad_clip 1.0 \ + --seed 42 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.0 \ + --no-output_bias \ + --output_init_std "${OUTPUT_INIT_STD}" \ + --norm_type rmsnorm \ + --qk_norm \ + --model_type ddit_elf \ + --ddit_mlp_type gelu \ + --state_format prob \ + --bridge dirichlet \ + --target_loss "${TARGET_LOSS}" \ + --loss_t_weight_mode "${LOSS_T_WEIGHT_MODE}" \ + --loss_t_min_weight "${LOSS_T_MIN_WEIGHT}" \ + --rollout_train_prob "${ROLLOUT_TRAIN_PROB}" \ + --rollout_train_rule "${ROLLOUT_TRAIN_RULE}" \ + --rollout_train_time_mode "${ROLLOUT_TRAIN_TIME_MODE}" \ + --rollout_train_steps "${ROLLOUT_TRAIN_STEPS}" \ + --rollout_train_steps_min "${ROLLOUT_TRAIN_STEPS_MIN}" \ + --rollout_train_infer_steps "${ROLLOUT_TRAIN_INFER_STEPS}" \ + --rollout_train_s_dist "${ROLLOUT_TRAIN_S_DIST}" \ + --rollout_train_s_min_frac "${ROLLOUT_TRAIN_S_MIN_FRAC}" \ + --rollout_train_s_max_frac "${ROLLOUT_TRAIN_S_MAX_FRAC}" \ + --rollout_train_temp "${ROLLOUT_TRAIN_TEMP}" \ + --rollout_train_max_gamma "${ROLLOUT_TRAIN_MAX_GAMMA}" \ + --rollout_train_corrupt_only \ + --rollout_train_samplewise \ + --rollout_train_selected_only \ + --no-rollout_train_compute_always \ + --rollout_train_sync_t \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --t_sampling_mode "${T_SAMPLING_MODE}" \ + --t_sampling_gumbel_loc "${T_GUMBEL_LOC}" \ + --t_sampling_gumbel_scale "${T_GUMBEL_SCALE}" \ + --t_sampling_logit_mean "${T_LOGIT_MEAN}" \ + --t_sampling_logit_std "${T_LOGIT_STD}" \ + --t_sampling_eps 1e-4 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --mask_mixture_original_prob 0.0 \ + --mask_mixture_lowk_prob 0.0 \ + --mask_mixture_lowcorrupt_prob 0.0 \ + --mask_mixture_block_prob 0.0 \ + --mask_mixture_all_prob "${MASK_MIXTURE_ALL_PROB}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 1024 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + --allow_tf32 \ + --activation_checkpointing \ + --activation_checkpoint_scope mlp \ + --ddp_gradient_as_bucket_view \ + 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/launch_lta_owt_unigramwrong_a0p7_8gpu.sh b/LTA_openwebtext_dualt/scripts/launch_lta_owt_unigramwrong_a0p7_8gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..adb8073467b6ba6cff4c9d356d090350094beb32 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/launch_lta_owt_unigramwrong_a0p7_8gpu.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} +export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} + +RUN_NAME=${RUN_NAME:-lta_owt_gpt2cached_len1024_unigramwrong_a0p7_shared10_ddit768x12_muon_ema_gbs512_8gpu_1m_$(date +%Y%m%d_%H%M%S)} +SAVE_DIR=${SAVE_DIR:-runs/${RUN_NAME}} +LOG_DIR=${LOG_DIR:-logs/unigramwrong_8gpu} +mkdir -p "${LOG_DIR}" + +OWT_CACHE=${OWT_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k} +UNIGRAM_COUNTS=${UNIGRAM_COUNTS:-${OWT_CACHE}/unigram_counts_1mchunks_i64.pt} + +python -m torch.distributed.run --nproc_per_node=8 --master_port="${MASTER_PORT:-31993}" train.py \ + --data_path /e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext \ + --openwebtext_split train_minus_100k \ + --text_column text \ + --detokenizer auto \ + --tokenizer_path /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode stream \ + --owt_cached_chunks \ + --owt_chunk_cache_dir "${OWT_CACHE}" \ + --max_len 1024 \ + --batch_size "${PER_GPU_BATCH_SIZE:-32}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE:-512}" \ + --num_workers "${NUM_WORKERS:-8}" \ + --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR:-4}" \ + --total_steps "${TOTAL_STEPS:-1000000}" \ + --warmup_steps "${WARMUP_STEPS:-2000}" \ + --log_every "${LOG_EVERY:-50}" \ + --eval_every "${EVAL_EVERY:-0}" \ + --save_every "${SAVE_EVERY:-50000}" \ + --latest_every "${LATEST_EVERY:-1000}" \ + --optimizer muon \ + --lr "${LEARNING_RATE:-0.002}" \ + --lr_schedule constant_warmup \ + --min_lr "${MIN_LR:-0}" \ + --weight_decay "${WEIGHT_DECAY:-0.0}" \ + --adam_beta1 0.9 \ + --adam_beta2 0.95 \ + --adam_eps 1e-8 \ + --muon_momentum 0.95 \ + --muon_ns_steps 5 \ + --muon_update_scale 1.0 \ + --ema_decay "${EMA_DECAY:-0.9999}" \ + --ema_start_step "${EMA_START_STEP:-0}" \ + --grad_clip 1.0 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.0 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --categorical_wrong_corpus_unigram_path "${UNIGRAM_COUNTS}" \ + --categorical_wrong_corpus_unigram_alpha "${UNIGRAM_ALPHA:-0.7}" \ + --categorical_wrong_unigram_prob "${WRONG_UNIGRAM_PROB:-0.70}" \ + --categorical_wrong_uniform_prob "${WRONG_UNIFORM_PROB:-0.20}" \ + --categorical_wrong_unigram_shared_prob "${WRONG_UNIGRAM_SHARED_PROB:-0.10}" \ + --simplex_bridge_sampler dirichlet \ + --infer_steps "${INFER_STEPS:-128}" \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + --allow_tf32 \ + --ddp_gradient_as_bucket_view \ + --bf16 \ + 2>&1 | tee "${LOG_DIR}/${RUN_NAME}.log" diff --git a/LTA_openwebtext_dualt/scripts/official_mauve_lm1b_export_features.py b/LTA_openwebtext_dualt/scripts/official_mauve_lm1b_export_features.py new file mode 100644 index 0000000000000000000000000000000000000000..3fc88046cfc8b8d23e9b0e88001350b3b391269b --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/official_mauve_lm1b_export_features.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import json +import re +import sys +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import AutoModel, AutoTokenizer + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from eval import build_model_from_ckpt +from flowtext_lab.data import WrappedStreamingTextSequenceDataset +from flowtext_lab.decode import model_time_for_step, sample_noise_simplex, state_for_model +from flowtext_lab.tokenization import BpeTextTokenizer + + +CKPT_PATH = Path( + "runs/lta_lm1b_duo_aligned_dirichlet_true_dualtline_flmpack_onehot_hardce_ddit_small_len128_gbs512_4xh20_1m/latest.pt" +) +TOKENIZER_PATH = "/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json" +DATA_PATH = "data/lm1b_test_parquet" +GPT2_LARGE = "/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard" + +N_SAMPLES = 5000 +MAX_LEN = 128 +STEPS = 128 +DECODE_BATCH = 32 +FEATURE_BATCH = 8 +FEATURE_MAX_LEN = 1024 +ENDPOINT_TEMP = 1.40 +FINAL_FROM = "state" +EPS = 1e-8 +SEED = 20260503 + +OUT_TEXT = Path("runs/decode_lab/mauve_official_lm1b_step_latest_t1p40_raw_5000.jsonl") +OUT_FEAT = Path("runs/decode_lab/mauve_official_lm1b_step_latest_t1p40_raw_5000_gpt2large_features.npz") + +SPECIAL_RE = re.compile(r"\s+") + + +def strip_common_special(text: str) -> str: + # Saved only for inspection; official MAUVE below uses raw text. + text = text.replace("[CLS]", " ").replace("[SEP]", " ").replace("[PAD]", " ") + text = text.replace("<|endoftext|>", " ") + return SPECIAL_RE.sub(" ", text).strip() + + +@torch.no_grad() +def decode_generated(device: torch.device) -> tuple[int, list[str]]: + tok = BpeTextTokenizer.from_file(TOKENIZER_PATH) + ckpt = torch.load(CKPT_PATH, map_location="cpu", weights_only=False) + step = int(ckpt.get("step", -1)) + ckpt_args = ckpt.get("args", {}) + target_prob = float(ckpt_args.get("target_prob", 1.0)) + dirichlet_concentration = float(ckpt_args.get("dirichlet_concentration_min", 1.0)) + model = build_model_from_ckpt(ckpt, tok.vocab_size, MAX_LEN, device).eval() + + torch.manual_seed(SEED) + generated: list[str] = [] + while len(generated) < N_SAMPLES: + bs = min(DECODE_BATCH, N_SAMPLES - len(generated)) + probs = sample_noise_simplex( + (bs, MAX_LEN), + tok.vocab_size, + device, + EPS, + noise_mode="dirichlet", + target_prob=target_prob, + noise_sigma=-1.0, + dirichlet_concentration=dirichlet_concentration, + ) + attn = torch.ones((bs, MAX_LEN), dtype=torch.bool, device=device) + last_endpoint = probs + for k in range(STEPS): + t = model_time_for_step("flow", k, STEPS, bs, device, dtype=torch.float32) + logits = model(state_for_model(model, probs, EPS), t, attn).float() + endpoint = F.softmax(logits / ENDPOINT_TEMP, dim=-1) + last_endpoint = endpoint + s = k / max(STEPS, 1) + t_next = (k + 1) / max(STEPS, 1) + gamma = min((t_next - s) / max(1.0 - s, EPS), 1.0) + probs = probs + gamma * (endpoint - probs) + probs = probs.clamp_min(EPS) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(EPS) + final_probs = last_endpoint if FINAL_FROM == "endpoint" else probs + ids = final_probs.argmax(dim=-1).detach().cpu().tolist() + generated.extend(tok.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in ids) + print(f"[decode] {len(generated)}/{N_SAMPLES}", flush=True) + + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return step, generated + + +def load_references() -> list[str]: + tok = BpeTextTokenizer.from_file(TOKENIZER_PATH) + ds = WrappedStreamingTextSequenceDataset(DATA_PATH, tok, MAX_LEN) + refs: list[str] = [] + for item in ds: + refs.append(tok.decode(item["ids"].tolist(), stop_at_eos=False, skip_special_tokens=False)) + if len(refs) >= N_SAMPLES: + break + return refs + + +@torch.no_grad() +def featurize(texts: list[str], name: str, device: torch.device) -> np.ndarray: + tokenizer = AutoTokenizer.from_pretrained(GPT2_LARGE) + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.pad_token_id = tokenizer.eos_token_id + model = AutoModel.from_pretrained(GPT2_LARGE, pad_token_id=tokenizer.eos_token_id).to(device).eval() + chunks: list[np.ndarray] = [] + t0 = time.time() + for start in range(0, len(texts), FEATURE_BATCH): + batch = texts[start : start + FEATURE_BATCH] + enc = tokenizer( + batch, + return_tensors="pt", + padding=True, + truncation=True, + max_length=FEATURE_MAX_LEN, + return_attention_mask=True, + ).to(device) + out = model( + input_ids=enc["input_ids"], + attention_mask=enc["attention_mask"], + output_hidden_states=True, + return_dict=True, + ) + hidden = out.hidden_states[-1] + last_idx = enc["attention_mask"].sum(dim=1) - 1 + feat = hidden[torch.arange(hidden.size(0), device=device), last_idx] + chunks.append(feat.float().cpu().numpy()) + if (start // FEATURE_BATCH) % 25 == 0: + print(f"[features] {name} {start + len(batch)}/{len(texts)}", flush=True) + arr = np.concatenate(chunks, axis=0) + print(f"[features] {name} {arr.shape} time={time.time() - t0:.1f}s", flush=True) + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return arr + + +def main() -> None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + OUT_TEXT.parent.mkdir(parents=True, exist_ok=True) + step, generated = decode_generated(device) + references = load_references() + print(f"[loaded] step={step} refs={len(references)} gen={len(generated)}", flush=True) + + with OUT_TEXT.open("w", encoding="utf-8") as f: + f.write( + json.dumps( + { + "type": "meta", + "checkpoint": str(CKPT_PATH), + "step": step, + "n_samples": N_SAMPLES, + "max_len": MAX_LEN, + "steps": STEPS, + "endpoint_temp": ENDPOINT_TEMP, + "final_from": FINAL_FROM, + "feature_max_len": FEATURE_MAX_LEN, + }, + ensure_ascii=False, + ) + + "\n" + ) + for i, text in enumerate(references): + f.write(json.dumps({"type": "reference", "index": i, "raw_text": text}, ensure_ascii=False) + "\n") + for i, text in enumerate(generated): + f.write( + json.dumps( + { + "type": "generated", + "index": i, + "raw_text": text, + "stripped_text": strip_common_special(text), + }, + ensure_ascii=False, + ) + + "\n" + ) + + ref_feat = featurize(references, "reference_raw", device) + gen_feat = featurize(generated, "generated_raw", device) + np.savez_compressed( + OUT_FEAT, + ref_raw=ref_feat, + gen_raw=gen_feat, + step=np.asarray([step], dtype=np.int64), + ) + print(f"[done] text={OUT_TEXT}", flush=True) + print(f"[done] features={OUT_FEAT}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/owt_decode_sweep_plan_20260503.sh b/LTA_openwebtext_dualt/scripts/owt_decode_sweep_plan_20260503.sh new file mode 100644 index 0000000000000000000000000000000000000000..2d6967d376f001891d217b508cd70c57e1a8e772 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/owt_decode_sweep_plan_20260503.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +RUN_DIR=runs/lta_owt_duo_aligned_dirichlet_true_dualtline_flmpack_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m +CKPT=${CKPT:-$RUN_DIR/latest.pt} +TOK=/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json +OUT_DIR=runs/decode_lab/owt_natural_sweep_20260503 +mkdir -p "$OUT_DIR" +GPU=${GPU:-3} +COMMON=(--checkpoint "$CKPT" --tokenizer_path "$TOK" --target_prob 1.0 --decode_batch_size 4 --topk 4 --seed 20260503) +# 1) strict training-support unconditional, full context sanity +CUDA_VISIBLE_DEVICES=$GPU python3 scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py "${COMMON[@]}" \ + --max_len 1024 --steps 1024 --restarts 4 --model_t_mode flow --noise_init dirichlet --dirichlet_init_concentration 1.0 \ + --prompts "" --config_set best_flowmap \ + --output "$OUT_DIR/uncond_strict_dirichlet_len1024_k1024.jsonl" +# 2) shorter diagnostic, broad update rules, strict prior +for MT in flow const05 const1 random; do + CUDA_VISIBLE_DEVICES=$GPU python3 scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py "${COMMON[@]}" \ + --max_len 128 --steps 256 --restarts 8 --model_t_mode "$MT" --noise_init dirichlet --dirichlet_init_concentration 1.0 \ + --prompts "" --config_set broad \ + --output "$OUT_DIR/uncond_dirichlet_len128_k256_${MT}_broad.jsonl" +done +# 3) off-support but useful diagnostic: logistic-normal init +for MT in flow const05 const1 random; do + CUDA_VISIBLE_DEVICES=$GPU python3 scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py "${COMMON[@]}" \ + --max_len 128 --steps 256 --restarts 8 --model_t_mode "$MT" --noise_init logistic_normal --target_prob 0.99 --noise_sigma -1.0 \ + --prompts "" --config_set broad \ + --output "$OUT_DIR/uncond_logistic_len128_k256_${MT}_broad.jsonl" +done +# 4) prompt continuation, no GPT2 BOS/EOT prepend, to check local fluency under context +for MT in flow const05 const1; do + CUDA_VISIBLE_DEVICES=$GPU python3 scripts/_tmp_flowtext_decode_lab_gpt2_prompt.py "${COMMON[@]}" \ + --max_len 256 --steps 256 --restarts 6 --model_t_mode "$MT" --noise_init dirichlet --dirichlet_init_concentration 1.0 \ + --prompts "The company said|In a recent report|Scientists have discovered|The government announced" --config_set broad \ + --output "$OUT_DIR/prompt_dirichlet_len256_k256_${MT}_broad.jsonl" +done diff --git a/LTA_openwebtext_dualt/scripts/probe_lm1b_nosep_pos0.py b/LTA_openwebtext_dualt/scripts/probe_lm1b_nosep_pos0.py new file mode 100644 index 0000000000000000000000000000000000000000..ea1b35c9487004c47f82cf65b1dd7dd6542b10d1 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/probe_lm1b_nosep_pos0.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from eval import build_model_from_ckpt +from flowtext_lab.data import EosPadCollator, WrappedStreamingTextSequenceDataset +from flowtext_lab.decode import state_for_model +from flowtext_lab.tokenization import BpeTextTokenizer +from scripts.apple_to_apple_lta_checks import ( + compact_piece, + make_bridge_for_eval, + masked_loss_acc, + token_piece, +) + + +@torch.no_grad() +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--checkpoint", required=True) + ap.add_argument("--out_json", required=True) + ap.add_argument("--data_path", default="data/lm1b_train_parquet") + ap.add_argument( + "--tokenizer_path", + default="/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json", + ) + ap.add_argument("--max_len", type=int, default=1024) + ap.add_argument("--batch_size", type=int, default=8) + ap.add_argument("--t_values", default="0.0,0.0078125,0.03125,0.125,0.5,1.0") + ap.add_argument("--force_mask_ratio", type=float, default=1.0) + ap.add_argument("--seed", type=int, default=20260525) + ap.add_argument("--eps", type=float, default=1e-8) + ap.add_argument("--cpu", action="store_true") + args = ap.parse_args() + + tok = BpeTextTokenizer.from_file(args.tokenizer_path) + device = torch.device("cuda" if torch.cuda.is_available() and not args.cpu else "cpu") + ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) + ckpt_args = dict(ckpt.get("args", {})) + model = build_model_from_ckpt(ckpt, tok.vocab_size, args.max_len, device).eval() + + ds = WrappedStreamingTextSequenceDataset( + args.data_path, + tok, + max_len=args.max_len, + detokenizer="auto", + wrap_mode="stream", + stream_add_eos_separator=False, + ) + loader = DataLoader(ds, batch_size=args.batch_size, collate_fn=EosPadCollator(tok.eos_id, max_len=args.max_len)) + batch = next(iter(loader)) + ids = batch["ids"].to(device) + attn = batch.get("attn_mask", torch.ones_like(ids, dtype=torch.bool)).to(device) + bos_id = int(tok.bos_id) + eos_id = int(tok.eos_id) + + rows = [] + for t_value in [float(x) for x in args.t_values.split(",") if x.strip()]: + torch.manual_seed(args.seed + int(round(t_value * 1_000_000))) + bridge = make_bridge_for_eval( + ids=ids, + attn=attn, + ckpt_args=ckpt_args, + vocab_size=tok.vocab_size, + t_value=t_value, + force_mask_ratio=args.force_mask_ratio, + eps=args.eps, + ) + logits = model(state_for_model(model, bridge.state, args.eps), bridge.t, attn).float() + probs0 = F.softmax(logits[:, 0, :], dim=-1) + pred0 = probs0.argmax(dim=-1) + counts = Counter(int(x) for x in pred0.detach().cpu().tolist()) + bos_prob = probs0[:, bos_id] + bos_rank = (probs0 > bos_prob[:, None]).sum(dim=-1) + 1 + eos_prob = probs0[:, eos_id] + topv, topi = probs0.mean(dim=0).topk(10) + rows.append( + { + "checkpoint": args.checkpoint, + "ckpt_step": int(ckpt.get("step", -1)), + "t": t_value, + "force_mask_ratio": args.force_mask_ratio, + "pos0_gold_id": int(ids[0, 0].detach().cpu()), + "pos0_gold_piece": compact_piece(token_piece(tok, int(ids[0, 0].detach().cpu()))), + "pos0_bos_id": bos_id, + "pos0_bos_piece": compact_piece(token_piece(tok, bos_id)), + "pos0_bos_prob_mean": float(bos_prob.mean().detach().cpu()), + "pos0_bos_prob_min": float(bos_prob.min().detach().cpu()), + "pos0_bos_prob_max": float(bos_prob.max().detach().cpu()), + "pos0_bos_rank_mean": float(bos_rank.float().mean().detach().cpu()), + "pos0_eos_prob_mean": float(eos_prob.mean().detach().cpu()), + "pos0_argmax_top": [ + {"id": i, "piece": compact_piece(token_piece(tok, i)), "count": c, "rate": c / max(ids.size(0), 1)} + for i, c in counts.most_common(10) + ], + "pos0_meanprob_top": [ + {"id": int(i), "piece": compact_piece(token_piece(tok, int(i))), "prob": float(v)} + for v, i in zip(topv.detach().cpu(), topi.detach().cpu()) + ], + "valid": masked_loss_acc(logits, ids, attn), + "pos0_corrupt_frac": float(bridge.corrupt_mask[:, 0].float().mean().detach().cpu()), + "pos0_wrong_frac": float(bridge.wrong_mask[:, 0].float().mean().detach().cpu()), + } + ) + + out = Path(args.out_json) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(rows, indent=2, ensure_ascii=False), encoding="utf-8") + for r in rows: + print( + f"step={r['ckpt_step']} t={r['t']:.4f} " + f"bos_prob={r['pos0_bos_prob_mean']:.4g} bos_rank={r['pos0_bos_rank_mean']:.1f} " + f"argmax={r['pos0_argmax_top'][0]['piece']}:{r['pos0_argmax_top'][0]['rate']:.2f}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_nosep_save1k_with_gumbel_watch.sh b/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_nosep_save1k_with_gumbel_watch.sh new file mode 100644 index 0000000000000000000000000000000000000000..a93444132bae4856c1c717dc4597a5012c70799d --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_nosep_save1k_with_gumbel_watch.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +MAX_LEN="${MAX_LEN:-1024}" +VOCAB_SIZE="${VOCAB_SIZE:-30522}" +CMIN="${CMIN:-${VOCAB_SIZE}}" +CMAX="${CMAX:-61044}" + +RUN_NAME="${RUN_NAME:-lta_lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_nosep_gbs512_4gpu_20k_save1k_gumbelwatch_${DATE_TAG}}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +SCORER="${SCORER:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard}" + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-4}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-32671}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-4}" +TOTAL_STEPS="${TOTAL_STEPS:-20000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +NUM_WORKERS="${NUM_WORKERS:-0}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +WRAP_STREAM_EOS_SEPARATOR="${WRAP_STREAM_EOS_SEPARATOR:-0}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +RESUME_PATH="${RESUME_PATH:-}" + +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}" + +WATCH_ENABLED="${WATCH_ENABLED:-1}" +WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-3}" +WATCH_STEP_INTERVAL="${WATCH_STEP_INTERVAL:-1000}" +WATCH_SLEEP_SECONDS="${WATCH_SLEEP_SECONDS:-30}" +WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}" +WATCH_STEPS="${WATCH_STEPS:-128}" +WATCH_ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP:-1.45}" +WATCH_ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P:-0.95}" +WATCH_GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START:-1.0}" +WATCH_GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END:-0.2}" +WATCH_DECODE_BATCH="${WATCH_DECODE_BATCH:-2}" +WATCH_SCORE_BATCH="${WATCH_SCORE_BATCH:-1}" +WATCH_SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH:-1024}" +WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_nosep_every1k_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P//./p}_tau${WATCH_GUMBEL_TAU_START//./p}_to_${WATCH_GUMBEL_TAU_END//./p}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}" +WATCH_LOG_DIR="${WATCH_LOG_DIR:-logs/lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_gumbel_sde_watch}" +WATCH_SCRIPT="${WATCH_SCRIPT:-logs/${RUN_NAME}_watcher.sh}" +WATCH_LOG="${WATCH_LOG:-logs/${RUN_NAME}_watcher.log}" +WATCH_PID_FILE="${WATCH_PID_FILE:-logs/${RUN_NAME}_watcher.pid}" + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Set RESUME_PATH, use a new RUN_NAME/SAVE_DIR, or ALLOW_EXISTING_SAVE_DIR=1." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" "${WATCH_LOG_DIR}" "${WATCH_OUT_BASE}" + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +WRAP_STREAM_ARGS=() +if [[ "${WRAP_STREAM_EOS_SEPARATOR}" == "0" || "${WRAP_STREAM_EOS_SEPARATOR}" == "false" || "${WRAP_STREAM_EOS_SEPARATOR}" == "False" ]]; then + WRAP_STREAM_ARGS+=(--wrap_stream_no_eos_separator) +fi + +write_watcher() { + cat > "${WATCH_SCRIPT}" <<'WATCH_EOF' +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +: "${RUN_DIR:?RUN_DIR is required}" +: "${OUT_BASE:?OUT_BASE is required}" +: "${LOG_DIR:?LOG_DIR is required}" +: "${TOKENIZER_PATH:?TOKENIZER_PATH is required}" +: "${SCORER:?SCORER is required}" + +RUN_STEM="$(basename "${RUN_DIR}")" +TEMP_TAG="${ENDPOINT_TEMP//./p}" +PROCESSED_FILE="${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMIN}_${CMAX}_gumbel_t${TEMP_TAG}_n${N_SAMPLES}.txt" + +mkdir -p "${OUT_BASE}" "${LOG_DIR}" +touch "${PROCESSED_FILE}" + +echo "[watch-gumbel] run_dir=${RUN_DIR}" +echo "[watch-gumbel] out_base=${OUT_BASE}" +echo "[watch-gumbel] interval=${STEP_INTERVAL} max_len=${MAX_LEN} steps=${STEPS} c=${CMIN}->${CMAX} temp=${ENDPOINT_TEMP} top_p=${ENDPOINT_TOP_P} tau=${GUMBEL_TAU_START}->${GUMBEL_TAU_END} n=${N_SAMPLES}" + +while true; do + shopt -s nullglob + ckpts=("${RUN_DIR}"/step_*.pt) + shopt -u nullglob + + if (( ${#ckpts[@]} == 0 )); then + echo "[watch-gumbel] $(date +%F_%T) no ckpt yet" + sleep "${SLEEP_SECONDS}" + continue + fi + + printf "%s\n" "${ckpts[@]}" | sort | while read -r ckpt; do + base="$(basename "${ckpt}")" + step="${base#step_}" + step="${step%.pt}" + step_num=$((10#${step})) + if (( step_num % STEP_INTERVAL != 0 )); then + continue + fi + if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then + continue + fi + + out_dir="${OUT_BASE}/step_${step}" + log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}.log" + mkdir -p "${out_dir}" + + echo "[watch-gumbel] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}" + CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" python scripts/eval_lm1b_c1024_fullycoupled_sde_genppl.py \ + --checkpoint "${ckpt}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --scorer "${SCORER}" \ + --out_dir "${out_dir}" \ + --n_samples "${N_SAMPLES}" \ + --max_len "${MAX_LEN}" \ + --steps "${STEPS}" \ + --batch_size "${DECODE_BATCH}" \ + --score_batch "${SCORE_BATCH}" \ + --score_max_length "${SCORE_MAX_LENGTH}" \ + --concentration_min "${CMIN}" \ + --concentration_max "${CMAX}" \ + --endpoint_temp "${ENDPOINT_TEMP}" \ + --endpoint_projection gumbel_softmax \ + --endpoint_top_p "${ENDPOINT_TOP_P}" \ + --gumbel_tau_start "${GUMBEL_TAU_START}" \ + --gumbel_tau_end "${GUMBEL_TAU_END}" \ + --model_t_mode support_t \ + --mean_mode endpoint_only \ + --semantic_power 1.0 \ + --noise_init dirichlet \ + --noise_dirichlet_concentration "${CMIN}" \ + --sde_resample dirichlet \ + --final_from blend_0.5 \ + --seed 20260524 \ + 2>&1 | tee -a "${log_file}" + + echo "${ckpt}" >> "${PROCESSED_FILE}" + echo "[watch-gumbel] $(date +%F_%T) done step_${step}" | tee -a "${log_file}" + done + + sleep "${SLEEP_SECONDS}" +done +WATCH_EOF + chmod +x "${WATCH_SCRIPT}" +} + +start_watcher() { + if [[ "${WATCH_ENABLED}" != "1" ]]; then + echo "[launch] watcher disabled" + return + fi + if [[ -f "${WATCH_PID_FILE}" ]]; then + old_pid="$(cat "${WATCH_PID_FILE}" || true)" + if [[ -n "${old_pid}" ]] && kill -0 "${old_pid}" 2>/dev/null; then + echo "[launch] watcher already running pid=${old_pid}" + return + fi + fi + write_watcher + nohup env \ + RUN_DIR="${SAVE_DIR}" \ + OUT_BASE="${WATCH_OUT_BASE}" \ + LOG_DIR="${WATCH_LOG_DIR}" \ + TOKENIZER_PATH="${TOKENIZER_PATH}" \ + SCORER="${SCORER}" \ + WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" \ + N_SAMPLES="${WATCH_N_SAMPLES}" \ + MAX_LEN="${MAX_LEN}" \ + STEPS="${WATCH_STEPS}" \ + DECODE_BATCH="${WATCH_DECODE_BATCH}" \ + SCORE_BATCH="${WATCH_SCORE_BATCH}" \ + SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH}" \ + CMIN="${CMIN}" \ + CMAX="${CMAX}" \ + ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP}" \ + ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P}" \ + GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START}" \ + GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END}" \ + STEP_INTERVAL="${WATCH_STEP_INTERVAL}" \ + SLEEP_SECONDS="${WATCH_SLEEP_SECONDS}" \ + bash "${WATCH_SCRIPT}" > "${WATCH_LOG}" 2>&1 & + echo "$!" > "${WATCH_PID_FILE}" + echo "[launch] watcher pid=$(cat "${WATCH_PID_FILE}") log=${WATCH_LOG}" +} + +echo "[launch] run=${RUN_NAME}" +echo "[launch] max_len=${MAX_LEN} gbs=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE} total_steps=${TOTAL_STEPS}" +echo "[launch] dirichlet C=${CMIN}->${CMAX} wrong_floor=${CATEGORICAL_WRONG_PROB_FLOOR}" +echo "[launch] wrap_stream_eos_separator=${WRAP_STREAM_EOS_SEPARATOR}" +start_watcher + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + --openwebtext_split all \ + --detokenizer auto \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + "${WRAP_STREAM_ARGS[@]}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --categorical_wrong_prob_floor "${CATEGORICAL_WRONG_PROB_FLOOR}" \ + --dirichlet_concentration_min "${CMIN}" \ + --dirichlet_concentration_max "${CMAX}" \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_save1k_with_gumbel_watch.sh b/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_save1k_with_gumbel_watch.sh new file mode 100644 index 0000000000000000000000000000000000000000..261ae812412b6069a57ae923378bae38c5218515 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_lm1b_dirichlet_len1024_Cv_to_2v_4gpu_save1k_with_gumbel_watch.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +MAX_LEN="${MAX_LEN:-1024}" +VOCAB_SIZE="${VOCAB_SIZE:-30522}" +CMIN="${CMIN:-${VOCAB_SIZE}}" +CMAX="${CMAX:-61044}" + +RUN_NAME="${RUN_NAME:-lta_lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_${DATE_TAG}}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +SCORER="${SCORER:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard}" + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-4}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-32671}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-4}" +TOTAL_STEPS="${TOTAL_STEPS:-20000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +NUM_WORKERS="${NUM_WORKERS:-0}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +RESUME_PATH="${RESUME_PATH:-}" + +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}" + +WATCH_ENABLED="${WATCH_ENABLED:-1}" +WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-3}" +WATCH_STEP_INTERVAL="${WATCH_STEP_INTERVAL:-1000}" +WATCH_SLEEP_SECONDS="${WATCH_SLEEP_SECONDS:-30}" +WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}" +WATCH_STEPS="${WATCH_STEPS:-128}" +WATCH_ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP:-1.45}" +WATCH_ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P:-0.95}" +WATCH_GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START:-1.0}" +WATCH_GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END:-0.2}" +WATCH_DECODE_BATCH="${WATCH_DECODE_BATCH:-2}" +WATCH_SCORE_BATCH="${WATCH_SCORE_BATCH:-1}" +WATCH_SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH:-1024}" +WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_every1k_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P//./p}_tau${WATCH_GUMBEL_TAU_START//./p}_to_${WATCH_GUMBEL_TAU_END//./p}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}" +WATCH_LOG_DIR="${WATCH_LOG_DIR:-logs/lm1b_dirichlet_len${MAX_LEN}_Cv_to_2v_gumbel_sde_watch}" +WATCH_SCRIPT="${WATCH_SCRIPT:-logs/${RUN_NAME}_watcher.sh}" +WATCH_LOG="${WATCH_LOG:-logs/${RUN_NAME}_watcher.log}" +WATCH_PID_FILE="${WATCH_PID_FILE:-logs/${RUN_NAME}_watcher.pid}" + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Set RESUME_PATH, use a new RUN_NAME/SAVE_DIR, or ALLOW_EXISTING_SAVE_DIR=1." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" "${WATCH_LOG_DIR}" "${WATCH_OUT_BASE}" + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +write_watcher() { + cat > "${WATCH_SCRIPT}" <<'WATCH_EOF' +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +: "${RUN_DIR:?RUN_DIR is required}" +: "${OUT_BASE:?OUT_BASE is required}" +: "${LOG_DIR:?LOG_DIR is required}" +: "${TOKENIZER_PATH:?TOKENIZER_PATH is required}" +: "${SCORER:?SCORER is required}" + +RUN_STEM="$(basename "${RUN_DIR}")" +TEMP_TAG="${ENDPOINT_TEMP//./p}" +PROCESSED_FILE="${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMIN}_${CMAX}_gumbel_t${TEMP_TAG}_n${N_SAMPLES}.txt" + +mkdir -p "${OUT_BASE}" "${LOG_DIR}" +touch "${PROCESSED_FILE}" + +echo "[watch-gumbel] run_dir=${RUN_DIR}" +echo "[watch-gumbel] out_base=${OUT_BASE}" +echo "[watch-gumbel] interval=${STEP_INTERVAL} max_len=${MAX_LEN} steps=${STEPS} c=${CMIN}->${CMAX} temp=${ENDPOINT_TEMP} top_p=${ENDPOINT_TOP_P} tau=${GUMBEL_TAU_START}->${GUMBEL_TAU_END} n=${N_SAMPLES}" + +while true; do + shopt -s nullglob + ckpts=("${RUN_DIR}"/step_*.pt) + shopt -u nullglob + + if (( ${#ckpts[@]} == 0 )); then + echo "[watch-gumbel] $(date +%F_%T) no ckpt yet" + sleep "${SLEEP_SECONDS}" + continue + fi + + printf "%s\n" "${ckpts[@]}" | sort | while read -r ckpt; do + base="$(basename "${ckpt}")" + step="${base#step_}" + step="${step%.pt}" + step_num=$((10#${step})) + if (( step_num % STEP_INTERVAL != 0 )); then + continue + fi + if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then + continue + fi + + out_dir="${OUT_BASE}/step_${step}" + log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}.log" + mkdir -p "${out_dir}" + + echo "[watch-gumbel] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}" + CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" python scripts/eval_lm1b_c1024_fullycoupled_sde_genppl.py \ + --checkpoint "${ckpt}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --scorer "${SCORER}" \ + --out_dir "${out_dir}" \ + --n_samples "${N_SAMPLES}" \ + --max_len "${MAX_LEN}" \ + --steps "${STEPS}" \ + --batch_size "${DECODE_BATCH}" \ + --score_batch "${SCORE_BATCH}" \ + --score_max_length "${SCORE_MAX_LENGTH}" \ + --concentration_min "${CMIN}" \ + --concentration_max "${CMAX}" \ + --endpoint_temp "${ENDPOINT_TEMP}" \ + --endpoint_projection gumbel_softmax \ + --endpoint_top_p "${ENDPOINT_TOP_P}" \ + --gumbel_tau_start "${GUMBEL_TAU_START}" \ + --gumbel_tau_end "${GUMBEL_TAU_END}" \ + --model_t_mode support_t \ + --mean_mode endpoint_only \ + --semantic_power 1.0 \ + --noise_init dirichlet \ + --noise_dirichlet_concentration "${CMIN}" \ + --sde_resample dirichlet \ + --final_from blend_0.5 \ + --seed 20260524 \ + 2>&1 | tee -a "${log_file}" + + echo "${ckpt}" >> "${PROCESSED_FILE}" + echo "[watch-gumbel] $(date +%F_%T) done step_${step}" | tee -a "${log_file}" + done + + sleep "${SLEEP_SECONDS}" +done +WATCH_EOF + chmod +x "${WATCH_SCRIPT}" +} + +start_watcher() { + if [[ "${WATCH_ENABLED}" != "1" ]]; then + echo "[launch] watcher disabled" + return + fi + if [[ -f "${WATCH_PID_FILE}" ]]; then + old_pid="$(cat "${WATCH_PID_FILE}" || true)" + if [[ -n "${old_pid}" ]] && kill -0 "${old_pid}" 2>/dev/null; then + echo "[launch] watcher already running pid=${old_pid}" + return + fi + fi + write_watcher + nohup env \ + RUN_DIR="${SAVE_DIR}" \ + OUT_BASE="${WATCH_OUT_BASE}" \ + LOG_DIR="${WATCH_LOG_DIR}" \ + TOKENIZER_PATH="${TOKENIZER_PATH}" \ + SCORER="${SCORER}" \ + WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" \ + N_SAMPLES="${WATCH_N_SAMPLES}" \ + MAX_LEN="${MAX_LEN}" \ + STEPS="${WATCH_STEPS}" \ + DECODE_BATCH="${WATCH_DECODE_BATCH}" \ + SCORE_BATCH="${WATCH_SCORE_BATCH}" \ + SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH}" \ + CMIN="${CMIN}" \ + CMAX="${CMAX}" \ + ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP}" \ + ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P}" \ + GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START}" \ + GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END}" \ + STEP_INTERVAL="${WATCH_STEP_INTERVAL}" \ + SLEEP_SECONDS="${WATCH_SLEEP_SECONDS}" \ + bash "${WATCH_SCRIPT}" > "${WATCH_LOG}" 2>&1 & + echo "$!" > "${WATCH_PID_FILE}" + echo "[launch] watcher pid=$(cat "${WATCH_PID_FILE}") log=${WATCH_LOG}" +} + +echo "[launch] run=${RUN_NAME}" +echo "[launch] max_len=${MAX_LEN} gbs=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE} total_steps=${TOTAL_STEPS}" +echo "[launch] dirichlet C=${CMIN}->${CMAX} wrong_floor=${CATEGORICAL_WRONG_PROB_FLOOR}" +start_watcher + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + --openwebtext_split all \ + --detokenizer auto \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --categorical_wrong_prob_floor "${CATEGORICAL_WRONG_PROB_FLOOR}" \ + --dirichlet_concentration_min "${CMIN}" \ + --dirichlet_concentration_max "${CMAX}" \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_falseanchor_floor_lognormal_gbs512_save1k.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_falseanchor_floor_lognormal_gbs512_save1k.sh new file mode 100644 index 0000000000000000000000000000000000000000..b31281f5c8e6a5aa22ad14a9e25004ecbb133459 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_falseanchor_floor_lognormal_gbs512_save1k.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +if [[ -n "${ACTIVATE_ENV:-}" ]]; then + source "${ACTIVATE_ENV}" +fi + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG_OVERRIDE:-WARN}" +export TORCH_NCCL_AVOID_RECORD_STREAMS="${TORCH_NCCL_AVOID_RECORD_STREAMS:-1}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +NNODES="${NNODES:-${MLP_WORKER_NUM:-2}}" +NODE_RANK="${NODE_RANK:-${MLP_ROLE_INDEX:-0}}" +MASTER_ADDR="${MASTER_ADDR:-${MLP_WORKER_0_HOST:-127.0.0.1}}" +MASTER_PORT="${MASTER_PORT:-${MLP_WORKER_0_PORT:-32831}}" +NPROC_PER_NODE="${NPROC_PER_NODE:-$(nvidia-smi --list-gpus | wc -l)}" + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + export CUDA_VISIBLE_DEVICES="$(seq -s, 0 $((NPROC_PER_NODE - 1)))" +fi + +WORLD_SIZE=$((NNODES * NPROC_PER_NODE)) +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-4}" + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-text}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" + +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-1024}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-10}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" + +LR="${LR:-3e-4}" +D_MODEL="${D_MODEL:-768}" +COND_DIM="${COND_DIM:-128}" +N_LAYERS="${N_LAYERS:-12}" +N_HEADS="${N_HEADS:-12}" +DIM_FF="${DIM_FF:-3072}" +DROPOUT="${DROPOUT:-0.1}" + +# Generation-mode finetune mixture: +# 60% all positions corrupted/generated, loss on all valid tokens. +# 40% original mask-ratio branch, preserving some true clean anchors. +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +MASK_MIXTURE_ORIGINAL_PROB="${MASK_MIXTURE_ORIGINAL_PROB:-0.4}" +MASK_MIXTURE_ALL_PROB="${MASK_MIXTURE_ALL_PROB:-0.6}" + +# Non-absorbing high-t endpoint: t=1 still has a wrong hard endpoint floor. +CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.03}" +CATEGORICAL_WRONG_FROM_FULL_VOCAB="${CATEGORICAL_WRONG_FROM_FULL_VOCAB:-1}" + +# Keep a late noise floor; do not mechanically sharpen the cloud. +LOGISTIC_NORMAL_SIGMA_MAX="${LOGISTIC_NORMAL_SIGMA_MAX:-3.0}" +LOGISTIC_NORMAL_SIGMA_MIN="${LOGISTIC_NORMAL_SIGMA_MIN:-0.5}" +LOGISTIC_NORMAL_TAU_MAX="${LOGISTIC_NORMAL_TAU_MAX:-1.0}" +LOGISTIC_NORMAL_TAU_MIN="${LOGISTIC_NORMAL_TAU_MIN:-1.0}" + +ABS_POS_EMBED="${ABS_POS_EMBED:-0}" +ABS_POS_INIT_STD="${ABS_POS_INIT_STD:-0.02}" +ACTIVATION_CHECKPOINTING="${ACTIVATION_CHECKPOINTING:-1}" +ACTIVATION_CHECKPOINT_SCOPE="${ACTIVATION_CHECKPOINT_SCOPE:-block}" +ACTIVATION_CHECKPOINT_INTERVAL="${ACTIVATION_CHECKPOINT_INTERVAL:-1}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" + +ABS_POS_ARGS=() +if [[ "${ABS_POS_EMBED}" == "1" ]]; then + ABS_POS_ARGS+=(--abs_pos_embed --abs_pos_init_std "${ABS_POS_INIT_STD}") +else + ABS_POS_ARGS+=(--no-abs_pos_embed) +fi + +CKPT_ARGS=() +if [[ "${ACTIVATION_CHECKPOINTING}" == "1" ]]; then + CKPT_ARGS+=(--activation_checkpointing --activation_checkpoint_scope "${ACTIVATION_CHECKPOINT_SCOPE}" --activation_checkpoint_interval "${ACTIVATION_CHECKPOINT_INTERVAL}") +fi + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +WRONG_SOURCE_ARGS=() +if [[ "${CATEGORICAL_WRONG_FROM_FULL_VOCAB}" == "1" ]]; then + WRONG_SOURCE_ARGS+=(--categorical_wrong_from_full_vocab) +fi + +RUN_ID="${RUN_ID:-${MLP_TASK_ID:-${MLP_JOB_ID:-${MLP_RUN_ID:-${MLP_WORKER_0_PORT:-$(date +%Y%m%d_%H%M%S)}}}}}" +RUN_ID="$(printf "%s" "${RUN_ID}" | tr -c 'A-Za-z0-9_-' '_')" +RUN_NAME="${RUN_NAME:-lta_owt_falseanchor_floor_lognormal_bert_c1024_len${MAX_LEN}_wrongfloor${CATEGORICAL_WRONG_PROB_FLOOR}_allp${MASK_MIXTURE_ALL_PROB}_sigmin${LOGISTIC_NORMAL_SIGMA_MIN}_taumin${LOGISTIC_NORMAL_TAU_MIN}_d${D_MODEL}_l${N_LAYERS}_gbs${GLOBAL_BATCH_SIZE}_${NNODES}node${NPROC_PER_NODE}gpu_${RUN_ID}}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_DIR="${LOG_DIR:-logs/owt_falseanchor_floor_lognormal_16gpu}" +LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.node${NODE_RANK}.log}" + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + exit 2 +fi + +mkdir -p "${LOG_DIR}" runs "${SAVE_DIR}" +exec > >(tee -a "${LOG_FILE}") 2>&1 + +echo "[launch] method=owt_falseanchor_floor_lognormal" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] node_rank=${NODE_RANK}/${NNODES} nproc=${NPROC_PER_NODE} world=${WORLD_SIZE} master=${MASTER_ADDR}:${MASTER_PORT}" +echo "[launch] len=${MAX_LEN} gbs=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE} total_steps=${TOTAL_STEPS}" +echo "[launch] mask original=${MASK_MIXTURE_ORIGINAL_PROB} all=${MASK_MIXTURE_ALL_PROB} ratio=${MIN_MASK_RATIO}->${MAX_MASK_RATIO}" +echo "[launch] wrong_floor=${CATEGORICAL_WRONG_PROB_FLOOR} wrong_full_vocab=${CATEGORICAL_WRONG_FROM_FULL_VOCAB}" +echo "[launch] lognormal sigma=${LOGISTIC_NORMAL_SIGMA_MAX}->${LOGISTIC_NORMAL_SIGMA_MIN} tau=${LOGISTIC_NORMAL_TAU_MAX}->${LOGISTIC_NORMAL_TAU_MIN}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr "${LR}" \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model "${D_MODEL}" \ + --cond_dim "${COND_DIM}" \ + --n_layers "${N_LAYERS}" \ + --n_heads "${N_HEADS}" \ + --dim_ff "${DIM_FF}" \ + --dropout "${DROPOUT}" \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --t_sampling_mode uniform \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --mask_mixture_original_prob "${MASK_MIXTURE_ORIGINAL_PROB}" \ + --mask_mixture_all_prob "${MASK_MIXTURE_ALL_PROB}" \ + "${ABS_POS_ARGS[@]}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_prob_floor "${CATEGORICAL_WRONG_PROB_FLOOR}" \ + "${WRONG_SOURCE_ARGS[@]}" \ + --simplex_bridge_sampler logistic_normal_linear_mean \ + --logistic_normal_sigma_max "${LOGISTIC_NORMAL_SIGMA_MAX}" \ + --logistic_normal_sigma_min "${LOGISTIC_NORMAL_SIGMA_MIN}" \ + --logistic_normal_tau_max "${LOGISTIC_NORMAL_TAU_MAX}" \ + --logistic_normal_tau_min "${LOGISTIC_NORMAL_TAU_MIN}" \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${CKPT_ARGS[@]}" \ + "${RESUME_ARGS[@]}" \ + --bf16 diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_lm1brecipe_elfLdim_gbs2048_save10k.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_lm1brecipe_elfLdim_gbs2048_save10k.sh new file mode 100644 index 0000000000000000000000000000000000000000..17ceeb0b1533d353e2acc473bb3591c997d425cd --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_16gpu_lm1brecipe_elfLdim_gbs2048_save10k.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +if [[ -n "${ACTIVATE_ENV:-}" ]]; then + source "${ACTIVATE_ENV}" +fi + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_NCCL_AVOID_RECORD_STREAMS="${TORCH_NCCL_AVOID_RECORD_STREAMS:-1}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +# MLP multi-node launcher variables. Override manually if running outside MLP. +NNODES="${NNODES:-${MLP_WORKER_NUM:-2}}" +NODE_RANK="${NODE_RANK:-${MLP_ROLE_INDEX:-0}}" +MASTER_ADDR="${MASTER_ADDR:-${MLP_WORKER_0_HOST:-127.0.0.1}}" +MASTER_PORT="${MASTER_PORT:-${MLP_WORKER_0_PORT:-32741}}" +NPROC_PER_NODE="${NPROC_PER_NODE:-$(nvidia-smi --list-gpus | wc -l)}" + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + export CUDA_VISIBLE_DEVICES="$(seq -s, 0 $((NPROC_PER_NODE - 1)))" +else + export CUDA_VISIBLE_DEVICES +fi + +WORLD_SIZE=$((NNODES * NPROC_PER_NODE)) +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-2048}" +# 739M full-vocab len1024 is activation-heavy; override to 16 only if memory is comfortably clear. +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-8}" +MICRO_BATCH=$((PER_GPU_BATCH_SIZE * WORLD_SIZE)) +GRAD_ACCUM=$(((GLOBAL_BATCH_SIZE + MICRO_BATCH - 1) / MICRO_BATCH)) + +if [[ "${WORLD_SIZE}" -le 0 || "${PER_GPU_BATCH_SIZE}" -le 0 || "${GLOBAL_BATCH_SIZE}" -le 0 ]]; then + echo "Invalid distributed/batch config: NNODES=${NNODES} NPROC_PER_NODE=${NPROC_PER_NODE} GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE} PER_GPU_BATCH_SIZE=${PER_GPU_BATCH_SIZE}" >&2 + exit 2 +fi +if (( MICRO_BATCH > GLOBAL_BATCH_SIZE )); then + echo "Warning: micro_batch=${MICRO_BATCH} > global_batch=${GLOBAL_BATCH_SIZE}; train.py may effectively use grad_accum=1." >&2 +fi + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-text}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" + +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-1024}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-10000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" + +# LM1B classic recipe defaults, with only the model dimensions moved to the +# ELF-L/huge scale. This keeps the DDiT/adaLN backbone, not ddit_elf. +LR="${LR:-3e-4}" +D_MODEL="${D_MODEL:-1280}" +COND_DIM="${COND_DIM:-128}" +N_LAYERS="${N_LAYERS:-32}" +N_HEADS="${N_HEADS:-16}" +DIM_FF="${DIM_FF:-5120}" +DROPOUT="${DROPOUT:-0.1}" + +ACTIVATION_CHECKPOINTING="${ACTIVATION_CHECKPOINTING:-1}" +ACTIVATION_CHECKPOINT_SCOPE="${ACTIVATION_CHECKPOINT_SCOPE:-block}" +ACTIVATION_CHECKPOINT_INTERVAL="${ACTIVATION_CHECKPOINT_INTERVAL:-1}" + +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}" + +if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then + ENABLE_TORCH_COMPILE=0 +fi + +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +CKPT_ARGS=() +if [[ "${ACTIVATION_CHECKPOINTING}" == "1" ]]; then + CKPT_ARGS+=(--activation_checkpointing --activation_checkpoint_scope "${ACTIVATION_CHECKPOINT_SCOPE}" --activation_checkpoint_interval "${ACTIVATION_CHECKPOINT_INTERVAL}") +fi + +if [[ ! -e "${DATA_PATH}" ]]; then + echo "Missing DATA_PATH: ${DATA_PATH}" >&2 + exit 2 +fi +if [[ ! -f "${TOKENIZER_PATH}" ]]; then + echo "Missing TOKENIZER_PATH: ${TOKENIZER_PATH}" >&2 + exit 2 +fi + +RUN_ID="${RUN_ID:-${MLP_TASK_ID:-${MLP_JOB_ID:-${MLP_RUN_ID:-${MLP_WORKER_0_PORT:-}}}}}" +if [[ -z "${RUN_NAME:-}" ]]; then + if [[ -z "${RUN_ID}" ]]; then + RUN_ID="$(date +%Y%m%d_%H%M%S)" + fi + RUN_ID="$(printf "%s" "${RUN_ID}" | tr -c 'A-Za-z0-9_-' '_')" + RUN_NAME="lta_owt_lm1bclassic_fullvocab_bert_c1024_len1024_elfLdim_d${D_MODEL}_l${N_LAYERS}_h${N_HEADS}_ff${DIM_FF}_lr${LR}_gbs${GLOBAL_BATCH_SIZE}_${NNODES}node${NPROC_PER_NODE}gpu_1m_save10k_${RUN_ID}" +fi +export RUN_NAME + +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_DIR="${LOG_DIR:-logs/owt_lm1bclassic_fullvocab_len1024_16gpu_elfLdim}" +LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.node${NODE_RANK}.log}" + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p "${LOG_DIR}" runs "${SAVE_DIR}" + +python - <<'PY' +import sys +import torch +import tokenizers +print(f"[launch] python={sys.executable}") +print(f"[launch] torch={torch.__version__} tokenizers={tokenizers.__version__}") +PY + +echo "[launch] method=owt_lm1bclassic_fullvocab_bert_c1024_len1024_16gpu_elfLdim" +echo "[launch] host=$(hostname) time=$(date -Iseconds)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" +echo "[launch] node_rank=${NODE_RANK}/${NNODES} nproc_per_node=${NPROC_PER_NODE} world_size=${WORLD_SIZE} master=${MASTER_ADDR}:${MASTER_PORT}" +echo "[launch] data_path=${DATA_PATH}" +echo "[launch] split=${OPENWEBTEXT_SPLIT} text_column=${TEXT_COLUMN}" +echo "[launch] tokenizer=${TOKENIZER_PATH}" +echo "[launch] max_len=${MAX_LEN} wrap=${WRAP_MODE} buffer=${WRAP_RECORD_BUFFER_SIZE}" +echo "[launch] model=ddit d${D_MODEL} layers=${N_LAYERS} heads=${N_HEADS} ff=${DIM_FF} cond=${COND_DIM} dropout=${DROPOUT} params≈739.65M" +echo "[launch] lm1b-classic recipe: lr=${LR} schedule=constant_warmup warmup=${WARMUP_STEPS} wd=0 beta2=0.999 bf16 mask=0.1->1.0 C=1024" +echo "[launch] global_batch=${GLOBAL_BATCH_SIZE} per_gpu_batch=${PER_GPU_BATCH_SIZE} micro_batch=${MICRO_BATCH} grad_accum≈${GRAD_ACCUM}" +echo "[launch] activation_checkpointing=${ACTIVATION_CHECKPOINTING} scope=${ACTIVATION_CHECKPOINT_SCOPE} interval=${ACTIVATION_CHECKPOINT_INTERVAL}" +echo "[launch] save_every=${SAVE_EVERY} latest_every=${LATEST_EVERY}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr "${LR}" \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model "${D_MODEL}" \ + --cond_dim "${COND_DIM}" \ + --n_layers "${N_LAYERS}" \ + --n_heads "${N_HEADS}" \ + --dim_ff "${DIM_FF}" \ + --dropout "${DROPOUT}" \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${CKPT_ARGS[@]}" \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_4gpu_blockanchors_save1k.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_4gpu_blockanchors_save1k.sh new file mode 100644 index 0000000000000000000000000000000000000000..027bee6e9d43dbd58d39fc689679084f83f8d194 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_classic_fullvocab_len1024_4gpu_blockanchors_save1k.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" + +# OWT classic full-vocab DDiT, context=1024, with explicit learned block anchors. +# The model inserts one non-target learned anchor token before every 128 data +# tokens internally, then gathers logits back to the original 1024 data slots. + +RUN_NAME="${RUN_NAME:-lta_owt_classic_fullvocab_bert_c1024_len1024_blockanchor128_abspos_d768_l12_gbs512_4gpu_1m_save1k_$(date +%Y%m%d_%H%M%S)}" +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +TEXT_COLUMN="${TEXT_COLUMN:-text}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-4}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-32741}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-4}" +TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +MAX_LEN="${MAX_LEN:-1024}" +BLOCK_ANCHOR_EVERY="${BLOCK_ANCHOR_EVERY:-128}" +ABS_POS_EMBED="${ABS_POS_EMBED:-1}" +ABS_POS_INIT_STD="${ABS_POS_INIT_STD:-0.02}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +NUM_WORKERS="${NUM_WORKERS:-0}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +RESUME_PATH="${RESUME_PATH:-}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +ENABLE_ACTIVATION_CHECKPOINTING="${ENABLE_ACTIVATION_CHECKPOINTING:-1}" +ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}" +FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}" + +if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then + ENABLE_TORCH_COMPILE=0 +fi + +COMPILE_ARGS=() +if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then + COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead) +fi + +ABS_POS_ARGS=() +if [[ "${ABS_POS_EMBED}" == "1" ]]; then + ABS_POS_ARGS+=(--abs_pos_embed --abs_pos_init_std "${ABS_POS_INIT_STD}") +else + ABS_POS_ARGS+=(--no-abs_pos_embed) +fi + +CHECKPOINT_ARGS=() +if [[ "${ENABLE_ACTIVATION_CHECKPOINTING}" == "1" ]]; then + CHECKPOINT_ARGS+=(--activation_checkpointing --activation_checkpoint_interval 1 --activation_checkpoint_scope block) +fi + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +if [[ ! -e "${DATA_PATH}" ]]; then + echo "Missing DATA_PATH: ${DATA_PATH}" >&2 + exit 2 +fi +if [[ ! -f "${TOKENIZER_PATH}" ]]; then + echo "Missing TOKENIZER_PATH: ${TOKENIZER_PATH}" >&2 + exit 2 +fi +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" + +WORLD_SIZE=$((NNODES * NPROC_PER_NODE)) +MICRO_BATCH=$((PER_GPU_BATCH_SIZE * WORLD_SIZE)) +GRAD_ACCUM=$(((GLOBAL_BATCH_SIZE + MICRO_BATCH - 1) / MICRO_BATCH)) + +echo "[launch] method=owt_classic_fullvocab_bert_c1024_len1024_blockanchor128 host=$(hostname) time=$(date -Iseconds)" +echo "[launch] run_name=${RUN_NAME}" +echo "[launch] save_dir=${SAVE_DIR}" +echo "[launch] log_file=${LOG_FILE}" +echo "[launch] data_path=${DATA_PATH}" +echo "[launch] split=${OPENWEBTEXT_SPLIT} text_column=${TEXT_COLUMN}" +echo "[launch] tokenizer=${TOKENIZER_PATH}" +echo "[launch] max_len=${MAX_LEN} block_anchor_every=${BLOCK_ANCHOR_EVERY} abs_pos_embed=${ABS_POS_EMBED} abs_pos_init_std=${ABS_POS_INIT_STD} wrap=${WRAP_MODE} buffer=${WRAP_RECORD_BUFFER_SIZE}" +echo "[launch] world_size=${WORLD_SIZE} per_gpu_batch=${PER_GPU_BATCH_SIZE} global_batch=${GLOBAL_BATCH_SIZE} grad_accum≈${GRAD_ACCUM}" +echo "[launch] classic recipe: lr=3e-4 constant_warmup warmup=2500 wd=0 beta2=0.999 dropout=0.1 bf16 mask=0.1->1.0 C=1024" +echo "[launch] activation_checkpointing=${ENABLE_ACTIVATION_CHECKPOINTING} save_every=${SAVE_EVERY} latest_every=${LATEST_EVERY} total_steps=${TOTAL_STEPS}" + +python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode same \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio 0.1 \ + --max_mask_ratio 1.0 \ + --block_anchor_every "${BLOCK_ANCHOR_EVERY}" \ + --block_anchor_init_std 0.02 \ + "${ABS_POS_ARGS[@]}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --dirichlet_concentration_min 1.0 \ + --dirichlet_concentration_max 1024.0 \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --simplex_bridge_sampler dirichlet \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${CHECKPOINT_ARGS[@]}" \ + "${RESUME_ARGS[@]}" \ + "${COMPILE_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_1m_mask1_save10k.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_1m_mask1_save10k.sh new file mode 100644 index 0000000000000000000000000000000000000000..10c95f711f6953f67b8818334cf54f3a4fa8a654 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_1m_mask1_save10k.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +export MASTER_PORT="${MASTER_PORT:-32683}" + +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +export PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +export TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +export WARMUP_STEPS="${WARMUP_STEPS:-2500}" +export SAVE_EVERY="${SAVE_EVERY:-10000}" +export LATEST_EVERY="${LATEST_EVERY:-1000}" +export LOG_EVERY="${LOG_EVERY:-100}" + +export MAX_LEN="${MAX_LEN:-1024}" +export VOCAB_SIZE="${VOCAB_SIZE:-30522}" +export CMIN="${CMIN:-${VOCAB_SIZE}}" +export CMAX="${CMAX:-61044}" + +# Only change for this control run: every valid token is corrupted/supervised. +export MIN_MASK_RATIO=1.0 +export MAX_MASK_RATIO=1.0 + +export CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}" + +# Keep the long 8-card training run from competing with itself unless enabled. +export WATCH_ENABLED="${WATCH_ENABLED:-0}" + +DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +export RUN_NAME="${RUN_NAME:-lta_owt_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_1m_save10k_${DATE_TAG}}" + +bash scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh.bak_specialpos_20260525_010736 b/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh.bak_specialpos_20260525_010736 new file mode 100644 index 0000000000000000000000000000000000000000..c949be098ec4c9e0127d0f32ad8d5b3aa39426ab --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh.bak_specialpos_20260525_010736 @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}" + +DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +MAX_LEN="${MAX_LEN:-1024}" +VOCAB_SIZE="${VOCAB_SIZE:-30522}" +CMIN="${CMIN:-${VOCAB_SIZE}}" +CMAX="${CMAX:-61044}" + +RUN_NAME="${RUN_NAME:-lta_owt_dirichlet_len${MAX_LEN}_Cv_to_2v_gbs512_8gpu_20k_save1k_gumbelwatch_${DATE_TAG}}" +SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}" + +DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}" +TEXT_COLUMN="${TEXT_COLUMN:-text}" +OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}" +SCORER="${SCORER:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard}" + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +NNODES="${NNODES:-1}" +NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-32681}" + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +TOTAL_STEPS="${TOTAL_STEPS:-20000}" +WARMUP_STEPS="${WARMUP_STEPS:-2500}" +LOG_EVERY="${LOG_EVERY:-100}" +SAVE_EVERY="${SAVE_EVERY:-1000}" +LATEST_EVERY="${LATEST_EVERY:-1000}" +EVAL_EVERY="${EVAL_EVERY:-0}" +NUM_WORKERS="${NUM_WORKERS:-0}" +WRAP_MODE="${WRAP_MODE:-stream}" +WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}" +ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}" +RESUME_PATH="${RESUME_PATH:-}" + +MIN_MASK_RATIO="${MIN_MASK_RATIO:-0.1}" +MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}" + +WATCH_ENABLED="${WATCH_ENABLED:-0}" +WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-7}" +WATCH_STEP_INTERVAL="${WATCH_STEP_INTERVAL:-1000}" +WATCH_SLEEP_SECONDS="${WATCH_SLEEP_SECONDS:-30}" +WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}" +WATCH_STEPS="${WATCH_STEPS:-128}" +WATCH_ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP:-1.45}" +WATCH_ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P:-0.95}" +WATCH_GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START:-1.0}" +WATCH_GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END:-0.2}" +WATCH_DECODE_BATCH="${WATCH_DECODE_BATCH:-2}" +WATCH_SCORE_BATCH="${WATCH_SCORE_BATCH:-1}" +WATCH_SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH:-1024}" +WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/owt_dirichlet_len${MAX_LEN}_Cv_to_2v_every1k_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P//./p}_tau${WATCH_GUMBEL_TAU_START//./p}_to_${WATCH_GUMBEL_TAU_END//./p}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}" +WATCH_LOG_DIR="${WATCH_LOG_DIR:-logs/owt_dirichlet_len${MAX_LEN}_Cv_to_2v_gumbel_sde_watch}" +WATCH_SCRIPT="${WATCH_SCRIPT:-logs/${RUN_NAME}_watcher.sh}" +WATCH_LOG="${WATCH_LOG:-logs/${RUN_NAME}_watcher.log}" +WATCH_PID_FILE="${WATCH_PID_FILE:-logs/${RUN_NAME}_watcher.pid}" + +if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then + echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2 + echo "Set RESUME_PATH, use a new RUN_NAME/SAVE_DIR, or ALLOW_EXISTING_SAVE_DIR=1." >&2 + exit 2 +fi + +mkdir -p logs runs "${SAVE_DIR}" "${WATCH_LOG_DIR}" "${WATCH_OUT_BASE}" + +RESUME_ARGS=() +if [[ -n "${RESUME_PATH}" ]]; then + RESUME_ARGS+=(--resume_path "${RESUME_PATH}") +fi + +TEXT_COLUMN_ARGS=() +if [[ -n "${TEXT_COLUMN}" ]]; then + TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}") +fi + +write_watcher() { + cat > "${WATCH_SCRIPT}" <<'WATCH_EOF' +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +: "${RUN_DIR:?RUN_DIR is required}" +: "${OUT_BASE:?OUT_BASE is required}" +: "${LOG_DIR:?LOG_DIR is required}" +: "${TOKENIZER_PATH:?TOKENIZER_PATH is required}" +: "${SCORER:?SCORER is required}" + +RUN_STEM="$(basename "${RUN_DIR}")" +TEMP_TAG="${ENDPOINT_TEMP//./p}" +PROCESSED_FILE="${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMIN}_${CMAX}_gumbel_t${TEMP_TAG}_n${N_SAMPLES}.txt" + +mkdir -p "${OUT_BASE}" "${LOG_DIR}" +touch "${PROCESSED_FILE}" + +echo "[watch-gumbel] run_dir=${RUN_DIR}" +echo "[watch-gumbel] out_base=${OUT_BASE}" +echo "[watch-gumbel] interval=${STEP_INTERVAL} max_len=${MAX_LEN} steps=${STEPS} c=${CMIN}->${CMAX} temp=${ENDPOINT_TEMP} top_p=${ENDPOINT_TOP_P} tau=${GUMBEL_TAU_START}->${GUMBEL_TAU_END} n=${N_SAMPLES}" + +while true; do + shopt -s nullglob + ckpts=("${RUN_DIR}"/step_*.pt) + shopt -u nullglob + + if (( ${#ckpts[@]} == 0 )); then + echo "[watch-gumbel] $(date +%F_%T) no ckpt yet" + sleep "${SLEEP_SECONDS}" + continue + fi + + printf "%s\n" "${ckpts[@]}" | sort | while read -r ckpt; do + base="$(basename "${ckpt}")" + step="${base#step_}" + step="${step%.pt}" + step_num=$((10#${step})) + if (( step_num % STEP_INTERVAL != 0 )); then + continue + fi + if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then + continue + fi + + out_dir="${OUT_BASE}/step_${step}" + log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}.log" + mkdir -p "${out_dir}" + + echo "[watch-gumbel] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}" + CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" python scripts/eval_lm1b_c1024_fullycoupled_sde_genppl.py \ + --checkpoint "${ckpt}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --scorer "${SCORER}" \ + --out_dir "${out_dir}" \ + --n_samples "${N_SAMPLES}" \ + --max_len "${MAX_LEN}" \ + --steps "${STEPS}" \ + --batch_size "${DECODE_BATCH}" \ + --score_batch "${SCORE_BATCH}" \ + --score_max_length "${SCORE_MAX_LENGTH}" \ + --concentration_min "${CMIN}" \ + --concentration_max "${CMAX}" \ + --endpoint_temp "${ENDPOINT_TEMP}" \ + --endpoint_projection gumbel_softmax \ + --endpoint_top_p "${ENDPOINT_TOP_P}" \ + --gumbel_tau_start "${GUMBEL_TAU_START}" \ + --gumbel_tau_end "${GUMBEL_TAU_END}" \ + --model_t_mode support_t \ + --mean_mode endpoint_only \ + --semantic_power 1.0 \ + --noise_init dirichlet \ + --noise_dirichlet_concentration "${CMIN}" \ + --sde_resample dirichlet \ + --final_from blend_0.5 \ + --seed 20260524 \ + 2>&1 | tee -a "${log_file}" + + echo "${ckpt}" >> "${PROCESSED_FILE}" + echo "[watch-gumbel] $(date +%F_%T) done step_${step}" | tee -a "${log_file}" + done + + sleep "${SLEEP_SECONDS}" +done +WATCH_EOF + chmod +x "${WATCH_SCRIPT}" +} + +start_watcher() { + if [[ "${WATCH_ENABLED}" != "1" ]]; then + echo "[launch] watcher disabled" + return + fi + if [[ -f "${WATCH_PID_FILE}" ]]; then + old_pid="$(cat "${WATCH_PID_FILE}" || true)" + if [[ -n "${old_pid}" ]] && kill -0 "${old_pid}" 2>/dev/null; then + echo "[launch] watcher already running pid=${old_pid}" + return + fi + fi + write_watcher + nohup env \ + RUN_DIR="${SAVE_DIR}" \ + OUT_BASE="${WATCH_OUT_BASE}" \ + LOG_DIR="${WATCH_LOG_DIR}" \ + TOKENIZER_PATH="${TOKENIZER_PATH}" \ + SCORER="${SCORER}" \ + WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" \ + N_SAMPLES="${WATCH_N_SAMPLES}" \ + MAX_LEN="${MAX_LEN}" \ + STEPS="${WATCH_STEPS}" \ + DECODE_BATCH="${WATCH_DECODE_BATCH}" \ + SCORE_BATCH="${WATCH_SCORE_BATCH}" \ + SCORE_MAX_LENGTH="${WATCH_SCORE_MAX_LENGTH}" \ + CMIN="${CMIN}" \ + CMAX="${CMAX}" \ + ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP}" \ + ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P}" \ + GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START}" \ + GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END}" \ + STEP_INTERVAL="${WATCH_STEP_INTERVAL}" \ + SLEEP_SECONDS="${WATCH_SLEEP_SECONDS}" \ + bash "${WATCH_SCRIPT}" > "${WATCH_LOG}" 2>&1 & + echo "$!" > "${WATCH_PID_FILE}" + echo "[launch] watcher pid=$(cat "${WATCH_PID_FILE}") log=${WATCH_LOG}" +} + +echo "[launch] run=${RUN_NAME}" +echo "[launch] data_path=${DATA_PATH} split=${OPENWEBTEXT_SPLIT} text_column=${TEXT_COLUMN}" +echo "[launch] max_len=${MAX_LEN} gbs=${GLOBAL_BATCH_SIZE} per_gpu=${PER_GPU_BATCH_SIZE} total_steps=${TOTAL_STEPS}" +echo "[launch] dirichlet C=${CMIN}->${CMAX} wrong_floor=${CATEGORICAL_WRONG_PROB_FLOOR}" +start_watcher + +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" python -m torch.distributed.run \ + --nnodes="${NNODES}" \ + --nproc_per_node="${NPROC_PER_NODE}" \ + --node_rank="${NODE_RANK}" \ + --master_addr="${MASTER_ADDR}" \ + --master_port="${MASTER_PORT}" \ + train.py \ + --data_path "${DATA_PATH}" \ + "${TEXT_COLUMN_ARGS[@]}" \ + --openwebtext_split "${OPENWEBTEXT_SPLIT}" \ + --detokenizer auto \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --save_dir "${SAVE_DIR}" \ + --wrap \ + --wrap_mode "${WRAP_MODE}" \ + --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \ + --max_len "${MAX_LEN}" \ + --batch_size "${PER_GPU_BATCH_SIZE}" \ + --num_workers "${NUM_WORKERS}" \ + --global_batch_size "${GLOBAL_BATCH_SIZE}" \ + --total_steps "${TOTAL_STEPS}" \ + --log_every "${LOG_EVERY}" \ + --eval_every "${EVAL_EVERY}" \ + --save_every "${SAVE_EVERY}" \ + --latest_every "${LATEST_EVERY}" \ + --lr 3e-4 \ + --weight_decay 0 \ + --adam_beta1 0.9 \ + --adam_beta2 0.999 \ + --adam_eps 1e-8 \ + --warmup_steps "${WARMUP_STEPS}" \ + --lr_schedule constant_warmup \ + --grad_clip 1.0 \ + --seed 123 \ + --d_model 768 \ + --cond_dim 128 \ + --n_layers 12 \ + --n_heads 12 \ + --dim_ff 3072 \ + --dropout 0.1 \ + --model_type ddit \ + --state_format prob \ + --bridge dirichlet \ + --target_loss hard_ce \ + --target_prob 1.0 \ + --min_t 0.0 \ + --max_t 1.0 \ + --dual_t \ + --corrupt_t_mode independent \ + --corrupt_min_t 0.0 \ + --corrupt_max_t 1.0 \ + --min_mask_ratio "${MIN_MASK_RATIO}" \ + --max_mask_ratio "${MAX_MASK_RATIO}" \ + --wrong_token_replace_prob 1.0 \ + --wrong_token_schedule linear_t \ + --wrong_token_exp_k 1.0 \ + --categorical_wrong_prob_floor "${CATEGORICAL_WRONG_PROB_FLOOR}" \ + --dirichlet_concentration_min "${CMIN}" \ + --dirichlet_concentration_max "${CMAX}" \ + --dirichlet_endpoint_mode categorical_dual_t \ + --dirichlet_semantic_t_mode same \ + --dirichlet_semantic_t_value 0.0 \ + --categorical_wrong_from_full_vocab \ + --eps 1e-8 \ + --infer_steps 128 \ + --decode_damping 1.0 \ + --max_gamma 1.0 \ + --decode_solver flowmap \ + --noise_init logistic_normal \ + --bridge_noise_init logistic_normal \ + --noise_sigma -1 \ + "${RESUME_ARGS[@]}" \ + --bf16 2>&1 | tee -a "${LOG_FILE}" diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_absrope_time4_dirichlet_len1025_Cv_to_2v_8gpu_1m_mask1_sameT_prebos_save10k.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_absrope_time4_dirichlet_len1025_Cv_to_2v_8gpu_1m_mask1_sameT_prebos_save10k.sh new file mode 100644 index 0000000000000000000000000000000000000000..f65c63b4d0f426cd3313ee0c0a88f8941eeb103e --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_absrope_time4_dirichlet_len1025_Cv_to_2v_8gpu_1m_mask1_sameT_prebos_save10k.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +# OWT through the existing ELF/T5 tokenized pipeline: +# [BOS] + up to 1024 existing T5 tokens. +# +# The ELF T5 records already end with , so we only prepend BOS here. +# T5 has no separate BOS token in this tokenizer; the repo wrapper falls back +# to bos_id = eos_id = . +# +# Backbone: +# ddit_elf = RMSNorm/SwiGLU/QK-norm + RoPE + 4 prefix time tokens. +# Learned absolute position embeddings are added before RoPE. +# +# Bridge: +# Dirichlet C=V->2V, mask_ratio=1.0, model t and corruption t are shared. + +export DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5}" +export TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json}" +export TOKENIZED_HF=1 +export TOKENIZED_PAD_TOKEN="${TOKENIZED_PAD_TOKEN:-pad}" +export TOKENIZED_PREPEND_BOS=1 +export TOKENIZED_APPEND_EOS=0 +export TOKENIZED_STRIP_EDGE_SPECIALS=0 + +export MAX_LEN="${MAX_LEN:-1025}" +export VOCAB_SIZE="${VOCAB_SIZE:-32100}" +export CMIN="${CMIN:-32100}" +export CMAX="${CMAX:-64200}" + +export MODEL_TYPE=ddit_elf +export ELF_NUM_TIME_TOKENS="${ELF_NUM_TIME_TOKENS:-4}" +export ELF_NUM_MODEL_MODE_TOKENS="${ELF_NUM_MODEL_MODE_TOKENS:-0}" +export QK_NORM="${QK_NORM:-1}" +export ABS_POS_EMBED=1 +export CORRUPT_T_MODE=same +export MIN_MASK_RATIO=1.0 +export MAX_MASK_RATIO=1.0 +export CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export NPROC_PER_NODE="${NPROC_PER_NODE:-8}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +export PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}" +export TOTAL_STEPS="${TOTAL_STEPS:-1000000}" +export WARMUP_STEPS="${WARMUP_STEPS:-2500}" +export SAVE_EVERY="${SAVE_EVERY:-10000}" +export LATEST_EVERY="${LATEST_EVERY:-1000}" +export LOG_EVERY="${LOG_EVERY:-100}" + +export DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +export RUN_NAME="${RUN_NAME:-lta_owt_t5elf_absrope_time4_dirichlet_len1025_Cv_to_2v_prebos_mask1_sameT_gbs512_b32_8gpu_1m_save10k_${DATE_TAG}}" + +export WATCH_ENABLED="${WATCH_ENABLED:-1}" +export WATCH_STEP_INTERVAL="${WATCH_STEP_INTERVAL:-10000}" +export WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}" +export WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-7}" +export WATCH_ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP:-1.45}" +export WATCH_ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P:-0.95}" +export WATCH_GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START:-1.0}" +export WATCH_GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END:-0.2}" +export WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/owt_t5elf_absrope_time4_len1025_Cv_to_2v_prebos_mask1_sameT_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P//./p}_tau${WATCH_GUMBEL_TAU_START//./p}_to_${WATCH_GUMBEL_TAU_END//./p}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}" +export WATCH_LOG_DIR="${WATCH_LOG_DIR:-logs/owt_t5elf_absrope_time4_len1025_Cv_to_2v_prebos_mask1_sameT_gumbel_sde_watch}" + +bash scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh diff --git a/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_8gpu_mask1_save1k_with_gumbel_watch.sh b/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_8gpu_mask1_save1k_with_gumbel_watch.sh new file mode 100644 index 0000000000000000000000000000000000000000..978a7666134a7649702ac38fb121de460fb076e9 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_8gpu_mask1_save1k_with_gumbel_watch.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5}" +export TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json}" +export TOKENIZED_HF="${TOKENIZED_HF:-1}" +export TOKENIZED_PAD_TOKEN="${TOKENIZED_PAD_TOKEN:-pad}" + +export VOCAB_SIZE="${VOCAB_SIZE:-32100}" +export CMIN="${CMIN:-32100}" +export CMAX="${CMAX:-64200}" + +export MIN_MASK_RATIO="${MIN_MASK_RATIO:-1.0}" +export MAX_MASK_RATIO="${MAX_MASK_RATIO:-1.0}" +export MASK_MIXTURE_ORIGINAL_PROB="${MASK_MIXTURE_ORIGINAL_PROB:-0.0}" +export MASK_MIXTURE_ALL_PROB="${MASK_MIXTURE_ALL_PROB:-0.0}" + +export DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}" +export RUN_NAME="${RUN_NAME:-lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_${DATE_TAG}}" +export TOTAL_STEPS="${TOTAL_STEPS:-20000}" +export SAVE_EVERY="${SAVE_EVERY:-1000}" +export LATEST_EVERY="${LATEST_EVERY:-1000}" +export WATCH_ENABLED="${WATCH_ENABLED:-1}" +export WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}" +export WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-7}" +export WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_every1k_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P:-0.95}_tau${WATCH_GUMBEL_TAU_START:-1.0}_to_${WATCH_GUMBEL_TAU_END:-0.2}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}" + +bash scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh diff --git a/LTA_openwebtext_dualt/scripts/run_train8_rollin_p50_long20k_4gpu.sh b/LTA_openwebtext_dualt/scripts/run_train8_rollin_p50_long20k_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..9e77b72a6a5c7e79f6ffb832f952a993f26d494e --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/run_train8_rollin_p50_long20k_4gpu.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt + +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +BASE_CACHE="${BASE_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks}" +TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}" +MAX_LEN="${MAX_LEN:-256}" +N_SAMPLES="${N_SAMPLES:-64}" +INFER_STEPS="${INFER_STEPS:-128}" +STEP_CHUNK="${STEP_CHUNK:-1000}" +MAX_TOTAL_STEPS="${MAX_TOTAL_STEPS:-20000}" +PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-128}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}" +RUN_NAME="${RUN_NAME:-train8_rollin_len256_rollin_p50_s4_i32_20260517_171654}" +GROUP_STAMP="${GROUP_STAMP:-$(date +%Y%m%d_%H%M%S)}" +OUT_ROOT="${OUT_ROOT:-docs/lta_samples/metrics_20260517/rollin_p50_long20k_len${MAX_LEN}_bs512_ode128_${GROUP_STAMP}}" +DRIVER_LOG="${DRIVER_LOG:-logs/rollin_p50_long20k_4gpu/${GROUP_STAMP}.log}" +CURVE_CSV="${CURVE_CSV:-${OUT_ROOT}/hit_ratio_curve.csv}" +mkdir -p "$(dirname "${DRIVER_LOG}")" "${OUT_ROOT}" + +cache="${BASE_CACHE}/gpt2_len${MAX_LEN}_train8_compact_overfit" +vocab_size="$( +python - "$cache" <<'PY' +import json +import sys +from pathlib import Path +meta = json.loads((Path(sys.argv[1]) / "meta.json").read_text()) +print(int(meta.get("compact_vocab_size", meta.get("vocab_size")))) +PY +)" + +if [[ ! -f "${CURVE_CSV}" ]]; then + echo "run_name,ckpt_step,train_views_seen,train_tokens_seen,token_acc_mean,exact_count,exact_ref_count,exact_ref_hits" > "${CURVE_CSV}" +fi + +latest_step() { + local run_name="$1" + python - "$run_name" <<'PY' +import re +import sys +from pathlib import Path +run = Path("runs") / sys.argv[1] +steps = [] +for path in run.glob("step_*.pt"): + m = re.search(r"step_(\d+)\.pt$", path.name) + if m: + steps.append(int(m.group(1))) +print(max(steps) if steps else 0) +PY +} + +free_port() { + python - <<'PY' +import socket +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + print(s.getsockname()[1]) +PY +} + +eval_latest() { + local target_step="$1" + local out_dir="${OUT_ROOT}/step_${target_step}" + mkdir -p "${out_dir}" + CUDA_VISIBLE_DEVICES="${EVAL_CUDA_VISIBLE_DEVICES:-0}" python scripts/eval_train8_decode_acc.py \ + --runs_glob "runs/${RUN_NAME}" \ + --data_dir "${cache}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --out_dir "${out_dir}" \ + --max_len "${MAX_LEN}" \ + --n_samples "${N_SAMPLES}" \ + --batch_size "${N_SAMPLES}" \ + --latest_only \ + --endpoint_softenings none \ + --steps "${INFER_STEPS}" \ + --decode_rule flowmap \ + --time_schedule logit_normal \ + --time_logit_mean -1.5 \ + --time_logit_std 0.8 \ + --model_t_mode post \ + --c_min 1 \ + --c_max 512 \ + --late_temp 1.0 \ + --final_from state \ + --final_decode argmax + python - "$out_dir" "$N_SAMPLES" "$GLOBAL_BATCH_SIZE" "$MAX_LEN" "$CURVE_CSV" "$RUN_NAME" <<'PY' +import json +import sys +from pathlib import Path +out = Path(sys.argv[1]) +n = int(sys.argv[2]) +global_batch = int(sys.argv[3]) +max_len = int(sys.argv[4]) +curve = Path(sys.argv[5]) +run_name = sys.argv[6] +row = json.loads((out / "decode_token_acc.jsonl").read_text().splitlines()[-1]) +views = int(row["ckpt_step"]) * global_batch +tokens = views * max_len +print( + "RESULT " + f"run={run_name} ckpt_step={row['ckpt_step']} views={views} " + f"token_acc={row['token_acc_mean']:.4f} exact={row['exact_count']}/{n} " + f"exact_refs={row['exact_ref_count']} hits={row['exact_ref_hits']}", + flush=True, +) +with curve.open("a", encoding="utf-8") as f: + f.write( + f"{run_name},{row['ckpt_step']},{views},{tokens},{row['token_acc_mean']}," + f"{row['exact_count']},{row['exact_ref_count']},\"{row['exact_ref_hits']}\"\n" + ) +PY +} + +echo "[rollin-p50-long20k] start stamp=${GROUP_STAMP} len=${MAX_LEN} vocab=${vocab_size} run=${RUN_NAME} out=${OUT_ROOT}" | tee -a "${DRIVER_LOG}" +while :; do + step_now="$(latest_step "${RUN_NAME}")" + if [[ "${step_now}" -ge "${MAX_TOTAL_STEPS}" ]]; then + echo "[rollin-p50-long20k] capped step=${step_now}" | tee -a "${DRIVER_LOG}" + break + fi + target_step=$((step_now + STEP_CHUNK)) + if [[ "${target_step}" -gt "${MAX_TOTAL_STEPS}" ]]; then + target_step="${MAX_TOTAL_STEPS}" + fi + resume_path="" + if [[ -f "runs/${RUN_NAME}/latest.pt" ]]; then + resume_path="runs/${RUN_NAME}/latest.pt" + fi + echo "[rollin-p50-long20k] train from=${step_now} to=${target_step}" | tee -a "${DRIVER_LOG}" + CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" \ + NPROC_PER_NODE="${NPROC_PER_NODE:-4}" \ + MASTER_PORT="$(free_port)" \ + OWT_CHUNK_CACHE_DIR="${cache}" \ + OWT_EXACT_REPEAT_PER_CHUNK="${OWT_EXACT_REPEAT_PER_CHUNK:-64}" \ + MAX_LEN="${MAX_LEN}" \ + VOCAB_SIZE_OVERRIDE="${vocab_size}" \ + D_MODEL="${D_MODEL:-192}" \ + COND_DIM="${COND_DIM:-64}" \ + N_LAYERS="${N_LAYERS:-3}" \ + N_HEADS="${N_HEADS:-3}" \ + DIM_FF="${DIM_FF:-768}" \ + TOTAL_STEPS="${target_step}" \ + PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE}" \ + GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE}" \ + NUM_WORKERS="${NUM_WORKERS:-0}" \ + LOG_EVERY="${LOG_EVERY:-100}" \ + SAVE_EVERY="${STEP_CHUNK}" \ + LATEST_EVERY="${STEP_CHUNK}" \ + WARMUP_STEPS="${WARMUP_STEPS:-10}" \ + LEARNING_RATE="${LEARNING_RATE:-0.002}" \ + WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" \ + MUON_IMPL="${MUON_IMPL:-legacy}" \ + OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" \ + MIN_MASK_RATIO=1.0 \ + MAX_MASK_RATIO=1.0 \ + MASK_MIXTURE_LOWK_PROB=0.0 \ + MASK_MIXTURE_ALL_PROB=1.0 \ + LOWK_CLEAN_TOKENS=0 \ + CLEAN_STATE_MODE=onehot \ + TARGET_LOSS=hard_ce \ + DIRICHLET_CONCENTRATION_MIN=1.0 \ + DIRICHLET_CONCENTRATION_MAX=1024 \ + SIMPLEX_BRIDGE_SAMPLER=dirichlet \ + CATEGORICAL_WRONG_PROB_FLOOR=0.0 \ + ROLLOUT_TRAIN_PROB=0.50 \ + ROLLOUT_TRAIN_STEPS=4 \ + ROLLOUT_TRAIN_INFER_STEPS=32 \ + ROLLOUT_TRAIN_TEMP=1.45 \ + ROLLOUT_TRAIN_MAX_GAMMA=1.0 \ + ROLLOUT_TRAIN_CORRUPT_ONLY=1 \ + ROLLOUT_TRAIN_SAMPLEWISE=1 \ + ROLLOUT_TRAIN_SELECTED_ONLY=1 \ + ROLLOUT_TRAIN_COMPUTE_ALWAYS=0 \ + ROLLOUT_TRAIN_SYNC_T=0 \ + RUN_NAME="${RUN_NAME}" \ + RESUME_PATH="${resume_path}" \ + bash scripts/launch_lta_owt_gpt2_softendpoint_mn_pilot_4gpu.sh + echo "[rollin-p50-long20k] eval step=${target_step}" | tee -a "${DRIVER_LOG}" + eval_latest "${target_step}" | tee -a "${DRIVER_LOG}" +done diff --git a/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_maskgit_entropy_push_20260507.py b/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_maskgit_entropy_push_20260507.py new file mode 100644 index 0000000000000000000000000000000000000000..943c3c6f2da9c7fe1cf0b9c070394f2b3ed510f7 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_maskgit_entropy_push_20260507.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +BASE_PATH = Path(__file__).with_name("sweep_categorical_c1024_maskgit_diffusion_20260506.py") +spec = importlib.util.spec_from_file_location("maskgit_base", BASE_PATH) +if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {BASE_PATH}") +base = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = base +spec.loader.exec_module(base) + + +def entropy_push_configs() -> list[base.DecodeCfg]: + C = base.DecodeCfg + return [ + C("linear_top64_t1p0_ep1p0", fill_curve="linear", top_k=64, sample_temp=1.0, endpoint_temp=1.0), + C("linear_top64_t1p2_ep1p3", fill_curve="linear", top_k=64, sample_temp=1.2, endpoint_temp=1.3), + C("linear_top128_t1p2_ep1p5", fill_curve="linear", top_k=128, sample_temp=1.2, endpoint_temp=1.5), + C("linear_top256_t1p5_ep1p5", fill_curve="linear", top_k=256, sample_temp=1.5, endpoint_temp=1.5), + C("linear_top512_t1p5_ep2p0", fill_curve="linear", top_k=512, sample_temp=1.5, endpoint_temp=2.0), + C("cosout_top128_t1p2_ep1p5", fill_curve="cosine_out", top_k=128, sample_temp=1.2, endpoint_temp=1.5), + C("coss_top128_t1p2_ep1p5", fill_curve="cosine_s", top_k=128, sample_temp=1.2, endpoint_temp=1.5), + C("pow1p5_top128_t1p2_ep1p5", fill_curve="pow1p5", top_k=128, sample_temp=1.2, endpoint_temp=1.5), + C("pow2_top128_t1p2_ep1p5", fill_curve="pow2", top_k=128, sample_temp=1.2, endpoint_temp=1.5), + C( + "linear_top128_t1p2_ep1p5_sel0p05", + fill_curve="linear", + top_k=128, + sample_temp=1.2, + endpoint_temp=1.5, + selection_noise=0.05, + ), + C( + "linear_top128_t1p2_ep1p5_final_endpoint", + fill_curve="linear", + top_k=128, + sample_temp=1.2, + endpoint_temp=1.5, + final_from="endpoint", + ), + ] + + +base.default_configs = entropy_push_configs + + +if __name__ == "__main__": + base.main() diff --git a/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_rolling_renoise_entropy_20260507.py b/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_rolling_renoise_entropy_20260507.py new file mode 100644 index 0000000000000000000000000000000000000000..d4d68fa4298fe65e438e44382bd62091fa02279d --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/sweep_categorical_c1024_rolling_renoise_entropy_20260507.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +BASE_PATH = Path(__file__).with_name("sweep_categorical_c1024_rolling_noise_20260506.py") +spec = importlib.util.spec_from_file_location("rolling_noise_base", BASE_PATH) +if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {BASE_PATH}") +base = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = base +spec.loader.exec_module(base) + + +def renoise_entropy_configs() -> list[base.DecodeCfg]: + C = base.DecodeCfg + return [ + # Full-state re-noise every step, no token commit. Final sampling temp is + # swept tightly around the transition where entropy rises before PPL blows up. + C("rn_t1p3_eta1_blend_ft1p25", endpoint_temp=1.3, final_sample=True, final_temp=1.25), + C("rn_t1p3_eta1_blend_ft1p35", endpoint_temp=1.3, final_sample=True, final_temp=1.35), + C("rn_t1p3_eta1_blend_ft1p45", endpoint_temp=1.3, final_sample=True, final_temp=1.45), + C("rn_t1p3_eta1_blend_ft1p55", endpoint_temp=1.3, final_sample=True, final_temp=1.55), + C("rn_t1p4_eta1_blend_ft1p30", endpoint_temp=1.4, final_sample=True, final_temp=1.30), + C("rn_t1p4_eta1_blend_ft1p40", endpoint_temp=1.4, final_sample=True, final_temp=1.40), + C("rn_t1p4_eta1_blend_ft1p50", endpoint_temp=1.4, final_sample=True, final_temp=1.50), + C("rn_t1p5_eta1_blend_ft1p30", endpoint_temp=1.5, final_sample=True, final_temp=1.30), + C("rn_t1p5_eta1_blend_ft1p40", endpoint_temp=1.5, final_sample=True, final_temp=1.40), + C("rn_t1p5_eta1_blend_ft1p50", endpoint_temp=1.5, final_sample=True, final_temp=1.50), + C( + "rn_sp0p85_sem1p5_t1p4_eta1_blend_ft1p40", + endpoint_temp=1.4, + support_power=0.85, + semantic_power=1.5, + final_sample=True, + final_temp=1.40, + ), + C( + "rn_sp1p15_sem1p8_t1p4_eta1_blend_ft1p40", + endpoint_temp=1.4, + support_power=1.15, + semantic_power=1.8, + final_sample=True, + final_temp=1.40, + ), + C("rn_t1p4_eta0p75_blend_ft1p40", endpoint_temp=1.4, roll_eta=0.75, final_sample=True, final_temp=1.40), + C("rn_t1p4_eta1_state_ft1p40", endpoint_temp=1.4, final_from="state", final_sample=True, final_temp=1.40), + ] + + +base.default_configs = renoise_entropy_configs + + +if __name__ == "__main__": + base.main() diff --git a/LTA_openwebtext_dualt/scripts/tmp_lm1b_linear_narrow_sweep.py b/LTA_openwebtext_dualt/scripts/tmp_lm1b_linear_narrow_sweep.py new file mode 100644 index 0000000000000000000000000000000000000000..685577e56f5deb063d3e1127028891dee3079f10 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/tmp_lm1b_linear_narrow_sweep.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path.cwd())) + +from eval import build_model_from_ckpt +from flowtext_lab.tokenization import BpeTextTokenizer +from scripts.flowtext_decode_lab import ( + DecodeConfig, + aggregate, + build_initial_state, + decode_batch, + decode_text, + text_metrics, +) + + +def main() -> None: + checkpoint = ( + "runs/lta_lm1b_duo_aligned_dirichlet_true_dualtline_flmpack_onehot_hardce_" + "ddit_small_len128_gbs512_4xh20_1m/latest.pt" + ) + tok_path = ( + "/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/" + "nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json" + ) + out_path = Path("runs/decode_lab/lm1b_latest_linear_narrow_raw_k512_uncond.jsonl") + out_path.parent.mkdir(parents=True, exist_ok=True) + + torch.manual_seed(202605011) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tok = BpeTextTokenizer.from_file(tok_path) + ckpt = torch.load(checkpoint, map_location="cpu") + model = build_model_from_ckpt(ckpt, tok.vocab_size, 128, device) + model.eval() + + configs = [] + for temp, damp, final in [ + (1.70, 0.70, "state"), + (1.80, 0.70, "blend"), + (1.85, 0.80, "state"), + (1.90, 0.80, "blend"), + (1.95, 0.90, "state"), + (2.00, 0.90, "blend"), + (2.05, 1.00, "state"), + (2.10, 0.80, "blend"), + (1.75, 1.00, "blend"), + ]: + label = f"linear_flowmap_t{str(temp).replace('.', 'p')}_d{str(damp).replace('.', 'p')}_{final}" + configs.append( + DecodeConfig( + label, + "flowmap", + steps=512, + model_t_mode="linear", + damping=damp, + max_gamma=1.0, + endpoint_temp=temp, + final_from=final, + ) + ) + + print(f"[info] device={device} ckpt={checkpoint} out={out_path} configs={len(configs)}", flush=True) + with out_path.open("w") as f: + for cfg in configs: + init, attn, lock, lock_probs, _ = build_initial_state( + tok, + [""], + 64, + 128, + 0.99, + 1e-8, + "dirichlet", + -1.0, + 1.0, + device, + ) + decoded = decode_batch(model, init, attn, lock, lock_probs, cfg, 1e-8, tok.eos_id) + ids = decoded.argmax(dim=-1).detach().cpu().tolist() + rows = [] + for i, row_ids in enumerate(ids): + text = decode_text(tok, row_ids) + metrics = text_metrics(text) + words = text.replace("[", " [").replace("]", " ]").split() + long_glued = sum(1 for word in words if len(word) >= 18 and word.isalpha()) + no_space_punct = text.count("##") + metrics["generic_weird_penalty"] = 0.08 * long_glued + 0.20 * no_space_punct + metrics["rank_score"] = float(metrics["quality"] - metrics["generic_weird_penalty"]) + metrics.update({"candidate": i, "text": text}) + rows.append(metrics) + + rows.sort(key=lambda row: row["rank_score"], reverse=True) + summary = { + "type": "summary", + "config": cfg.__dict__, + "agg": aggregate(rows), + "best_rank_score": rows[0]["rank_score"], + } + f.write(json.dumps(summary, ensure_ascii=False) + "\n") + for rank, row in enumerate(rows[:3], 1): + rec = {"type": "sample", "rank": rank, "config": cfg.__dict__, **row} + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + f.flush() + + print("\n" + "=" * 96, flush=True) + print("[config]", cfg.label, cfg.__dict__, flush=True) + print("[agg]", json.dumps(summary["agg"], ensure_ascii=False), "best_rank_score=", rows[0]["rank_score"], flush=True) + print("[sample]", rows[0]["text"], flush=True) + + print("[done]", out_path, flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/tmp_lm1b_update_rule_sweep.py b/LTA_openwebtext_dualt/scripts/tmp_lm1b_update_rule_sweep.py new file mode 100644 index 0000000000000000000000000000000000000000..07bf021a031a9d755091417ad09b481f2db7059d --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/tmp_lm1b_update_rule_sweep.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path.cwd())) + +from eval import build_model_from_ckpt +from flowtext_lab.tokenization import BpeTextTokenizer +from scripts.flowtext_decode_lab import ( + DecodeConfig, + aggregate, + build_initial_state, + decode_batch, + decode_text, + text_metrics, +) + + +def run_config(model, tok, cfg, out_file, device): + init, attn, lock, lock_probs, _ = build_initial_state( + tok, + [""], + 64, + 128, + 0.99, + 1e-8, + "dirichlet", + -1.0, + 1.0, + device, + ) + decoded = decode_batch(model, init, attn, lock, lock_probs, cfg, 1e-8, tok.eos_id) + ids = decoded.argmax(dim=-1).detach().cpu().tolist() + rows = [] + for i, row_ids in enumerate(ids): + text = decode_text(tok, row_ids) + metrics = text_metrics(text) + words = text.replace("[", " [").replace("]", " ]").split() + long_glued = sum(1 for word in words if len(word) >= 18 and word.isalpha()) + repeated_sep_like = max(0, text.count("[CLS]") - 1) + max(0, text.count("[SEP]") - 6) + metrics["generic_weird_penalty"] = 0.08 * long_glued + 0.03 * repeated_sep_like + metrics["rank_score"] = float(metrics["quality"] - metrics["generic_weird_penalty"]) + metrics.update({"candidate": i, "text": text}) + rows.append(metrics) + rows.sort(key=lambda row: row["rank_score"], reverse=True) + summary = { + "type": "summary", + "config": cfg.__dict__, + "agg": aggregate(rows), + "best_rank_score": rows[0]["rank_score"], + } + out_file.write(json.dumps(summary, ensure_ascii=False) + "\n") + for rank, row in enumerate(rows[:3], 1): + rec = {"type": "sample", "rank": rank, "config": cfg.__dict__, **row} + out_file.write(json.dumps(rec, ensure_ascii=False) + "\n") + out_file.flush() + print("\n" + "=" * 96, flush=True) + print("[config]", cfg.label, cfg.__dict__, flush=True) + print("[agg]", json.dumps(summary["agg"], ensure_ascii=False), "best_rank_score=", rows[0]["rank_score"], flush=True) + print("[sample]", rows[0]["text"], flush=True) + + +def main() -> None: + checkpoint = ( + "runs/lta_lm1b_duo_aligned_dirichlet_true_dualtline_flmpack_onehot_hardce_" + "ddit_small_len128_gbs512_4xh20_1m/latest.pt" + ) + tok_path = ( + "/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/" + "nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json" + ) + out_path = Path("runs/decode_lab/lm1b_latest_update_rules_raw_k512_uncond.jsonl") + out_path.parent.mkdir(parents=True, exist_ok=True) + torch.manual_seed(202605012) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tok = BpeTextTokenizer.from_file(tok_path) + ckpt = torch.load(checkpoint, map_location="cpu") + model = build_model_from_ckpt(ckpt, tok.vocab_size, 128, device) + model.eval() + + configs = [ + DecodeConfig("linear_replace_eta0p20_t1p6_state", "replace", steps=512, model_t_mode="linear", eta=0.20, endpoint_temp=1.60, final_from="state"), + DecodeConfig("linear_replace_eta0p35_t1p6_state", "replace", steps=512, model_t_mode="linear", eta=0.35, endpoint_temp=1.60, final_from="state"), + DecodeConfig("linear_replace_eta0p50_t1p6_state", "replace", steps=512, model_t_mode="linear", eta=0.50, endpoint_temp=1.60, final_from="state"), + DecodeConfig("linear_replace_eta0p50_t1p8_state", "replace", steps=512, model_t_mode="linear", eta=0.50, endpoint_temp=1.80, final_from="state"), + DecodeConfig("linear_replace_eta0p35_t1p8_blend", "replace", steps=512, model_t_mode="linear", eta=0.35, endpoint_temp=1.80, final_from="blend"), + DecodeConfig("linear_geometric_eta0p10_t1p6_state", "geometric", steps=512, model_t_mode="linear", eta=0.10, endpoint_temp=1.60, final_from="state"), + DecodeConfig("linear_geometric_eta0p20_t1p7_state", "geometric", steps=512, model_t_mode="linear", eta=0.20, endpoint_temp=1.70, final_from="state"), + DecodeConfig("linear_geometric_eta0p35_t1p8_state", "geometric", steps=512, model_t_mode="linear", eta=0.35, endpoint_temp=1.80, final_from="state"), + DecodeConfig("linear_centered_eta0p20_t1p7_state", "centered_residual", steps=512, model_t_mode="linear", eta=0.20, endpoint_temp=1.70, final_from="state"), + DecodeConfig("linear_centered_eta0p35_t1p8_state", "centered_residual", steps=512, model_t_mode="linear", eta=0.35, endpoint_temp=1.80, final_from="state"), + ] + + print(f"[info] device={device} ckpt={checkpoint} out={out_path} configs={len(configs)}", flush=True) + with out_path.open("w") as out_file: + for cfg in configs: + run_config(model, tok, cfg, out_file, device) + print("[done]", out_path, flush=True) + + +if __name__ == "__main__": + main() diff --git a/LTA_openwebtext_dualt/scripts/trace_train8_ode_decode_support.py b/LTA_openwebtext_dualt/scripts/trace_train8_ode_decode_support.py new file mode 100644 index 0000000000000000000000000000000000000000..6a74982ae3ff5e7bf071bf84c33ed7947ea70926 --- /dev/null +++ b/LTA_openwebtext_dualt/scripts/trace_train8_ode_decode_support.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python +from __future__ import annotations + +import argparse +import csv +import json +import math +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = REPO_ROOT / "scripts" +for path in (REPO_ROOT, SCRIPTS_DIR): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from flowtext_lab.decode import sample_noise_simplex, state_for_model # noqa: E402 +from flowtext_lab.tokenization import BpeTextTokenizer # noqa: E402 +from infer_context_compare_from_c128 import ( # noqa: E402 + build_model, + final_decode_ids, + make_time_grid, + soften_endpoint_with_prior, + temperature, +) + + +def load_refs(data_dir: Path, max_len: int) -> tuple[np.ndarray, dict[str, Any]]: + meta = json.loads((data_dir / "meta.json").read_text()) + n = int(meta.get("num_chunks", meta.get("n_chunks", 0))) + if n <= 0: + size = (data_dir / "chunks.i32.bin").stat().st_size // np.dtype(np.int32).itemsize + n = size // max_len + arr = np.memmap(data_dir / "chunks.i32.bin", dtype=np.int32, mode="r") + refs = np.asarray(arr).reshape(n, -1)[:, :max_len].copy() + return refs, meta + + +def compact_token_text(tokenizer: BpeTextTokenizer, meta: dict[str, Any], new_id: int) -> str: + id_map = meta.get("id_map_new_to_old") or {} + old_id = int(id_map.get(str(int(new_id)), int(new_id))) + return tokenizer.decode([old_id], stop_at_eos=False, skip_special_tokens=False).replace("\n", "\\n") + + +def token_match(ids: torch.Tensor, refs: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + # ids: [B, L], refs: [R, L] -> best_acc [B], best_ref [B] + matches = (ids[:, None, :] == refs[None, :, :]).float().mean(dim=-1) + best_acc, best_ref = matches.max(dim=1) + return best_acc, best_ref + + +def mean_gold_prob(probs: torch.Tensor, refs: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + # probs: [B, L, V], refs: [R, L] -> best average gold prob over refs. + vals = [] + for ref in refs: + idx = ref.unsqueeze(0).expand(probs.size(0), -1).unsqueeze(-1) + vals.append(probs.gather(-1, idx).squeeze(-1).mean(dim=-1)) + scores = torch.stack(vals, dim=1) + best_score, best_ref = scores.max(dim=1) + return best_score, best_ref + + +def entropy(probs: torch.Tensor, eps: float) -> torch.Tensor: + p = probs.clamp_min(eps) + return -(p * p.log()).sum(dim=-1) + + +def top_token_summary(ids: torch.Tensor, tokenizer: BpeTextTokenizer, meta: dict[str, Any], k: int = 5) -> list[dict[str, Any]]: + flat = ids.reshape(-1).detach().cpu() + counts = torch.bincount(flat, minlength=int(flat.max().item()) + 1).float() + total = float(counts.sum().item()) + out = [] + for tid in counts.topk(min(k, counts.numel())).indices.tolist(): + count = int(counts[tid].item()) + if count <= 0: + continue + out.append({ + "id": int(tid), + "old_text": compact_token_text(tokenizer, meta, int(tid)), + "count": count, + "mass": count / max(total, 1.0), + }) + return out + + +def flowmap_update(probs: torch.Tensor, endpoint: torch.Tensor, progress: float, next_progress: float, eps: float) -> torch.Tensor: + dt = max(float(next_progress) - float(progress), 0.0) + gamma = min(dt / max(1.0 - float(progress), eps), 1.0) + out = probs + gamma * (endpoint - probs) + out = out.clamp_min(eps) + return out / out.sum(dim=-1, keepdim=True).clamp_min(eps) + + +def lock_step_stats(state_steps: torch.Tensor, endpoint_steps: torch.Tensor, refs: torch.Tensor) -> dict[str, Any]: + # state_steps: [K+1, B, L], endpoint_steps: [K, B, L] + final_ids = state_steps[-1] + final_acc, final_ref = token_match(final_ids, refs) + K = endpoint_steps.size(0) + state_locks = [] + endpoint_firsts = [] + wrong_counts = [] + for b in range(final_ids.size(0)): + ref = refs[final_ref[b]] + wrong = final_ids[b] != ref + wrong_pos = wrong.nonzero(as_tuple=False).flatten() + wrong_counts.append(int(wrong_pos.numel())) + for pos in wrong_pos.tolist(): + tok = final_ids[b, pos] + matches = state_steps[:, b, pos] == tok + lock = None + for s in range(matches.numel()): + if bool(matches[s:].all()): + lock = s + break + if lock is not None: + state_locks.append(lock) + ep_matches = endpoint_steps[:, b, pos] == tok + ep_first = ep_matches.nonzero(as_tuple=False) + if ep_first.numel() > 0: + endpoint_firsts.append(int(ep_first[0, 0].item()) + 1) + + def summarize(values: list[int]) -> dict[str, float | int | None]: + if not values: + return {"n": 0, "min": None, "p25": None, "median": None, "p75": None, "max": None, "mean": None} + arr = np.asarray(values, dtype=np.float64) + return { + "n": int(arr.size), + "min": float(arr.min()), + "p25": float(np.quantile(arr, 0.25)), + "median": float(np.quantile(arr, 0.5)), + "p75": float(np.quantile(arr, 0.75)), + "max": float(arr.max()), + "mean": float(arr.mean()), + } + + return { + "final_token_acc_mean": float(final_acc.mean().detach().cpu()), + "final_exact_count": int((final_acc >= 1.0).sum().detach().cpu()), + "final_best_ref": final_ref.detach().cpu().tolist(), + "wrong_count_per_sample": wrong_counts, + "wrong_state_lock_step": summarize(state_locks), + "wrong_endpoint_first_emit_step": summarize(endpoint_firsts), + "num_steps": int(K), + } + + +@torch.inference_mode() +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--checkpoint", required=True) + ap.add_argument("--data_dir", required=True) + ap.add_argument("--tokenizer_path", required=True) + ap.add_argument("--out_dir", required=True) + ap.add_argument("--max_len", type=int, default=256) + ap.add_argument("--n_samples", type=int, default=64) + ap.add_argument("--steps", type=int, default=128) + ap.add_argument("--seed", type=int, default=20314773) + ap.add_argument("--time_schedule", default="logit_normal", choices=["uniform", "logit_normal", "power_low", "power_high"]) + ap.add_argument("--time_logit_mean", type=float, default=-1.5) + ap.add_argument("--time_logit_std", type=float, default=0.8) + ap.add_argument("--time_power", type=float, default=2.0) + ap.add_argument("--model_t_mode", default="post", choices=["pre", "post"]) + ap.add_argument("--early_temp", type=float, default=1.0) + ap.add_argument("--late_temp", type=float, default=1.0) + ap.add_argument("--temp_end", type=float, default=1.0) + ap.add_argument("--temp_power", type=float, default=1.0) + ap.add_argument("--endpoint_softening", default="none", choices=["none", "uniform"]) + ap.add_argument("--endpoint_soft_power", type=float, default=1.0) + ap.add_argument("--endpoint_soft_min_conf", type=float, default=0.0) + ap.add_argument("--endpoint_soft_max_conf", type=float, default=1.0) + ap.add_argument("--final_from", default="state", choices=["state", "endpoint", "blend"]) + ap.add_argument("--final_decode", default="argmax", choices=["argmax", "sample"]) + ap.add_argument("--pos_extend", default="repeat") + ap.add_argument("--snapshot_steps", default="1,2,4,8,16,32,64,96,128") + ap.add_argument("--eps", type=float, default=1e-8) + args = ap.parse_args() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path) + refs_np, meta = load_refs(Path(args.data_dir), args.max_len) + refs = torch.as_tensor(refs_np, dtype=torch.long, device=device) + ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False, mmap=True) + model = build_model(ckpt, tokenizer, args.max_len, device, args.pos_extend) + vocab_size = int(getattr(model, "vocab_size", tokenizer.vocab_size)) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + time_grid = make_time_grid( + args.steps, + schedule=args.time_schedule, + logit_mean=args.time_logit_mean, + logit_std=args.time_logit_std, + power=args.time_power, + seed=args.seed, + device=device, + ) + torch.manual_seed(args.seed) + probs = sample_noise_simplex( + (args.n_samples, args.max_len), + vocab_size, + device, + args.eps, + noise_mode="dirichlet", + target_prob=1.0, + noise_sigma=-1.0, + dirichlet_concentration=1.0, + ) + attn = torch.ones((args.n_samples, args.max_len), dtype=torch.bool, device=device) + ref_attn = torch.ones((refs.size(0), args.max_len), dtype=torch.bool, device=device) + oracle_onehot = F.one_hot(refs, num_classes=vocab_size).float() + + state_steps = [probs.argmax(dim=-1).detach().cpu()] + endpoint_steps = [] + step_rows: list[dict[str, Any]] = [] + snapshots: dict[str, Any] = {} + snapshot_steps = {int(x) for x in args.snapshot_steps.split(",") if x.strip()} + last_endpoint = probs + + for step in range(args.steps): + progress = float(time_grid[step].item()) + next_progress = float(time_grid[step + 1].item()) + if args.model_t_mode == "pre": + t = torch.full((args.n_samples,), progress, dtype=torch.float32, device=device) + ref_t = torch.full((refs.size(0),), progress, dtype=torch.float32, device=device) + else: + t = torch.full((args.n_samples,), next_progress, dtype=torch.float32, device=device) + ref_t = torch.full((refs.size(0),), next_progress, dtype=torch.float32, device=device) + temp = temperature(step, args.steps, args.early_temp, args.late_temp, args.temp_end, args.temp_power) + logits = model(state_for_model(model, probs, args.eps), t, attn).float() + raw_endpoint = F.softmax(logits / temp, dim=-1) + endpoint, endpoint_alpha = soften_endpoint_with_prior( + raw_endpoint, + next_progress, + mode=args.endpoint_softening, + power=args.endpoint_soft_power, + min_conf=args.endpoint_soft_min_conf, + max_conf=args.endpoint_soft_max_conf, + eps=args.eps, + ) + last_endpoint = endpoint + endpoint_ids = endpoint.argmax(dim=-1) + endpoint_steps.append(endpoint_ids.detach().cpu()) + + state_ids_before = probs.argmax(dim=-1) + state_acc_before, state_ref_before = token_match(state_ids_before, refs) + endpoint_acc, endpoint_ref = token_match(endpoint_ids, refs) + state_pgold, state_pgold_ref = mean_gold_prob(probs, refs) + endpoint_pgold, endpoint_pgold_ref = mean_gold_prob(endpoint, refs) + + oracle_logits = model(state_for_model(model, oracle_onehot, args.eps), ref_t, ref_attn).float() + oracle_endpoint = F.softmax(oracle_logits / temp, dim=-1) + oracle_ids = oracle_endpoint.argmax(dim=-1) + oracle_acc = (oracle_ids == refs).float().mean(dim=-1) + oracle_pgold = oracle_endpoint.gather(-1, refs.unsqueeze(-1)).squeeze(-1).mean(dim=-1) + + probs = flowmap_update(probs, endpoint, progress, next_progress, args.eps) + state_ids_after = probs.argmax(dim=-1) + state_acc_after, state_ref_after = token_match(state_ids_after, refs) + state_steps.append(state_ids_after.detach().cpu()) + + row = { + "step": step + 1, + "progress": progress, + "next_progress": next_progress, + "dt": next_progress - progress, + "gamma": min((next_progress - progress) / max(1.0 - progress, args.eps), 1.0), + "temperature": temp, + "endpoint_alpha": endpoint_alpha, + "state_acc_before_mean": float(state_acc_before.mean().detach().cpu()), + "state_acc_after_mean": float(state_acc_after.mean().detach().cpu()), + "endpoint_acc_mean": float(endpoint_acc.mean().detach().cpu()), + "state_exact_after": int((state_acc_after >= 1.0).sum().detach().cpu()), + "endpoint_exact": int((endpoint_acc >= 1.0).sum().detach().cpu()), + "state_pgold_best_mean": float(state_pgold.mean().detach().cpu()), + "endpoint_pgold_best_mean": float(endpoint_pgold.mean().detach().cpu()), + "state_entropy_mean": float(entropy(probs, args.eps).mean().detach().cpu()), + "endpoint_entropy_mean": float(entropy(endpoint, args.eps).mean().detach().cpu()), + "state_maxprob_mean": float(probs.max(dim=-1).values.mean().detach().cpu()), + "endpoint_maxprob_mean": float(endpoint.max(dim=-1).values.mean().detach().cpu()), + "oracle_clean_acc_mean": float(oracle_acc.mean().detach().cpu()), + "oracle_clean_pgold_mean": float(oracle_pgold.mean().detach().cpu()), + "state_best_ref_mode": int(torch.mode(state_ref_after).values.detach().cpu()), + "endpoint_best_ref_mode": int(torch.mode(endpoint_ref).values.detach().cpu()), + "state_best_ref_counts": torch.bincount(state_ref_after, minlength=refs.size(0)).detach().cpu().tolist(), + "endpoint_best_ref_counts": torch.bincount(endpoint_ref, minlength=refs.size(0)).detach().cpu().tolist(), + } + step_rows.append(row) + + if (step + 1) in snapshot_steps: + snapshots[str(step + 1)] = { + "state_top_tokens": top_token_summary(state_ids_after, tokenizer, meta), + "endpoint_top_tokens": top_token_summary(endpoint_ids, tokenizer, meta), + "state_best_acc": state_acc_after.detach().cpu().tolist(), + "endpoint_best_acc": endpoint_acc.detach().cpu().tolist(), + "state_best_ref": state_ref_after.detach().cpu().tolist(), + "endpoint_best_ref": endpoint_ref.detach().cpu().tolist(), + } + + if (step + 1) % 16 == 0 or step == 0: + print( + f"step={step+1} state_acc={row['state_acc_after_mean']:.4f} " + f"endpoint_acc={row['endpoint_acc_mean']:.4f} oracle_acc={row['oracle_clean_acc_mean']:.4f} " + f"state_exact={row['state_exact_after']} endpoint_exact={row['endpoint_exact']}", + flush=True, + ) + + if args.final_from == "state": + final = probs + elif args.final_from == "endpoint": + final = last_endpoint + else: + final = 0.5 * probs + 0.5 * last_endpoint + final_ids = final_decode_ids(final, mode=args.final_decode, temp=1.0, top_k=0, top_p=1.0, eps=args.eps) + final_acc, final_ref = token_match(final_ids, refs) + endpoint_steps_t = torch.stack(endpoint_steps, dim=0) + state_steps_t = torch.stack(state_steps, dim=0) + lock_stats = lock_step_stats(state_steps_t, endpoint_steps_t, refs.detach().cpu()) + + with (out_dir / "step_metrics.csv").open("w", newline="", encoding="utf-8") as f: + fieldnames = [ + "step", + "progress", + "next_progress", + "dt", + "gamma", + "temperature", + "endpoint_alpha", + "state_acc_before_mean", + "state_acc_after_mean", + "endpoint_acc_mean", + "state_exact_after", + "endpoint_exact", + "state_pgold_best_mean", + "endpoint_pgold_best_mean", + "state_entropy_mean", + "endpoint_entropy_mean", + "state_maxprob_mean", + "endpoint_maxprob_mean", + "oracle_clean_acc_mean", + "oracle_clean_pgold_mean", + "state_best_ref_mode", + "endpoint_best_ref_mode", + ] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in step_rows: + writer.writerow({k: row[k] for k in fieldnames}) + + summary = { + "checkpoint": args.checkpoint, + "data_dir": args.data_dir, + "max_len": args.max_len, + "n_samples": args.n_samples, + "steps": args.steps, + "seed": args.seed, + "time_schedule": args.time_schedule, + "time_logit_mean": args.time_logit_mean, + "time_logit_std": args.time_logit_std, + "model_t_mode": args.model_t_mode, + "final_token_acc_mean": float(final_acc.mean().detach().cpu()), + "final_token_acc_min": float(final_acc.min().detach().cpu()), + "final_token_acc_max": float(final_acc.max().detach().cpu()), + "final_exact_count": int((final_acc >= 1.0).sum().detach().cpu()), + "final_best_ref_counts": torch.bincount(final_ref, minlength=refs.size(0)).detach().cpu().tolist(), + "lock_stats": lock_stats, + "snapshots": snapshots, + "last_step": step_rows[-1] if step_rows else None, + } + (out_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + torch.save( + { + "state_steps": state_steps_t, + "endpoint_steps": endpoint_steps_t, + "final_ids": final_ids.detach().cpu(), + "refs": refs.detach().cpu(), + }, + out_dir / "trajectory_ids.pt", + ) + print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) + + +if __name__ == "__main__": + main()