boot log hgqj70cqc286v3
Browse files- logs/boot-hgqj70cqc286v3.log +1113 -0
logs/boot-hgqj70cqc286v3.log
CHANGED
|
@@ -20388,3 +20388,1116 @@ worker-comfyui - Websocket receive timed out. Still waiting...
|
|
| 20388 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20389 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20390 |
worker-comfyui - Websocket receive timed out. Still waiting...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20391 |
25%|██▌ | 5/20 [07:06<21:20, 85.39s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20392 |
30%|███ | 6/20 [08:31<19:55, 85.42s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20393 |
35%|███▌ | 7/20 [09:57<18:31, 85.50s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20388 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20389 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20390 |
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20391 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20392 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20393 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20394 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20395 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20396 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20397 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20398 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20399 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20400 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20401 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20402 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20403 |
+
[minimax] boot log shipped
|
| 20404 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20405 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20406 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20407 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20408 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20409 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20410 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20411 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20412 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20413 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20414 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20415 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20416 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20417 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20418 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20419 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20420 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20421 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20422 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20423 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20424 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20425 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20426 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20427 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20428 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20429 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20430 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20431 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20432 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20433 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20434 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20435 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20436 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20437 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20438 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20439 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20440 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20441 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20442 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20443 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20444 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20445 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20446 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20447 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20448 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20449 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20450 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20451 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20452 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20453 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20454 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20455 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20456 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20457 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20458 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20459 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20460 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20461 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20462 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20463 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20464 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20465 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20466 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20467 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20468 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20469 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20470 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20471 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20472 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20473 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20474 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20475 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20476 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20477 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20478 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20479 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20480 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20481 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20482 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20483 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20484 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20485 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20486 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20487 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20488 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20489 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20490 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20491 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20492 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20493 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20494 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20495 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20496 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20497 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20498 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20499 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20500 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20501 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20502 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20503 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20504 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20505 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20506 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20507 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20508 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20509 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20510 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20511 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20512 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20513 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20514 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20515 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20516 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20517 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20518 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20519 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20520 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20521 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20522 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20523 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20524 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20525 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20526 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20527 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20528 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20529 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20530 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20531 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20532 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20533 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20534 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20535 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20536 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20537 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20538 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20539 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20540 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20541 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20542 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20543 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20544 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20545 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20546 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20547 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20548 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20549 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20550 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20551 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20552 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20553 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20554 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20555 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20556 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20557 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20558 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20559 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20560 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20561 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20562 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20563 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20564 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20565 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20566 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20567 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20568 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20569 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20570 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20571 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20572 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20573 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20574 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20575 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20576 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20577 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20578 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20579 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20580 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20581 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20582 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20583 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20584 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20585 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20586 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20587 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20588 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20589 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20590 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20591 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20592 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20593 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20594 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20595 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20596 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20597 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20598 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20599 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20600 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20601 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20602 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20603 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20604 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20605 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20606 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20607 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20608 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20609 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20610 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20611 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20612 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20613 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20614 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20615 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20616 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20617 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20618 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20619 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20620 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20621 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20622 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20623 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20624 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20625 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20626 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51973 MB
|
| 20627 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20628 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=46524 MB
|
| 20629 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20630 |
+
|
| 20631 |
25%|██▌ | 5/20 [07:06<21:20, 85.39s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 20632 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x781d4b61d080: Actual Resident VRAM = 0 MB
|
| 20633 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780f14ef3440: Actual Resident VRAM = 160 MB
|
| 20634 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780ebb35b580: Actual Resident VRAM = 25888 MB
|
| 20635 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x7811c40107c0: Actual Resident VRAM = 20000 MB
|
| 20636 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 46048 MB
|
| 20637 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20638 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20639 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20640 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20641 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20642 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20643 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20644 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20645 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20646 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20647 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20648 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20649 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20650 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20651 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20652 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20653 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20654 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20655 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20656 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20657 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20658 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20659 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20660 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20661 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20662 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20663 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20664 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20665 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20666 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20667 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20668 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20669 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20670 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20671 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20672 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20673 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20674 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20675 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20676 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20677 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20678 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20679 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20680 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20681 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20682 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20683 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20684 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20685 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20686 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20687 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20688 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20689 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20690 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20691 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20692 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20693 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20694 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20695 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20696 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20697 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20698 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20699 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20700 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20701 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20702 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20703 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20704 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20705 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20706 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20707 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20708 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20709 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20710 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20711 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20712 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20713 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20714 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20715 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20716 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20717 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20718 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20719 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20720 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20721 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20722 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20723 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20724 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20725 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20726 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20727 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20728 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20729 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20730 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20731 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20732 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20733 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20734 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20735 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20736 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20737 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20738 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20739 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20740 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20741 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20742 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20743 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20744 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20745 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20746 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20747 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20748 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20749 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20750 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20751 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20752 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20753 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20754 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20755 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20756 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20757 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20758 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20759 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20760 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20761 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20762 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20763 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20764 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20765 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20766 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20767 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20768 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20769 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20770 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20771 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20772 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20773 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20774 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20775 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20776 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20777 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20778 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20779 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20780 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20781 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20782 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20783 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20784 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20785 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20786 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20787 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20788 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20789 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20790 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20791 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20792 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20793 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20794 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20795 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20796 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20797 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20798 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20799 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20800 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20801 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20802 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20803 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20804 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20805 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20806 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20807 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20808 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20809 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20810 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20811 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20812 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20813 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20814 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20815 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20816 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20817 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20818 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20819 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20820 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20821 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20822 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20823 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20824 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20825 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20826 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20827 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20828 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20829 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20830 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20831 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20832 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20833 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20834 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20835 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20836 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20837 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20838 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20839 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20840 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20841 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20842 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20843 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20844 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20845 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20846 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20847 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20848 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20849 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20850 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20851 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20852 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20853 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20854 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20855 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20856 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20857 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20858 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20859 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20860 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20861 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20862 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20863 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20864 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20865 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20866 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20867 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20868 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20869 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20870 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20871 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20872 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20873 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20874 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20875 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20876 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20877 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20878 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20879 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20880 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20881 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20882 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20883 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20884 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20885 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20886 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20887 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20888 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20889 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20890 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20891 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20892 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20893 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20894 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20895 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20896 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20897 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20898 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20899 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20900 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20901 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20902 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20903 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20904 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20905 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20906 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20907 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20908 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20909 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20910 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20911 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20912 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20913 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20914 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20915 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20916 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20917 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20918 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20919 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20920 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20921 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20922 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20923 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20924 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20925 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20926 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20927 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20928 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20929 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20930 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20931 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20932 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20933 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20934 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20935 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20936 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20937 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20938 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20939 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20940 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20941 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51973 MB
|
| 20942 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20943 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=46524 MB
|
| 20944 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20945 |
+
|
| 20946 |
30%|███ | 6/20 [08:31<19:55, 85.42s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 20947 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x781d4b61d080: Actual Resident VRAM = 0 MB
|
| 20948 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780f14ef3440: Actual Resident VRAM = 160 MB
|
| 20949 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780ebb35b580: Actual Resident VRAM = 25888 MB
|
| 20950 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x7811c40107c0: Actual Resident VRAM = 20000 MB
|
| 20951 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 46048 MB
|
| 20952 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20953 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20954 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20955 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20956 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20957 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20958 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20959 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20960 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20961 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20962 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20963 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20964 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20965 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20966 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20967 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20968 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20969 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20970 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20971 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20972 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20973 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20974 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20975 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20976 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20977 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20978 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20979 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20980 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20981 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20982 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20983 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20984 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 20985 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20986 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20987 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20988 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20989 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20990 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20991 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20992 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20993 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 20994 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20995 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20996 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 20997 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 20998 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 20999 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21000 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21001 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21002 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21003 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21004 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21005 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21006 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21007 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21008 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21009 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21010 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21011 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21012 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21013 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21014 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21015 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21016 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21017 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21018 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21019 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21020 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21021 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21022 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21023 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21024 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21025 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21026 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21027 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21028 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21029 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21030 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21031 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21032 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21033 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21034 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21035 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21036 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21037 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21038 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21039 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21040 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21041 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21042 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21043 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21044 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21045 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21046 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21047 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21048 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21049 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21050 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21051 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21052 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21053 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21054 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21055 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21056 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21057 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21058 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21059 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21060 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21061 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21062 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21063 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21064 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21065 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21066 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21067 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21068 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21069 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21070 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21071 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21072 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21073 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21074 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21075 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21076 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21077 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21078 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21079 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21080 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21081 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21082 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21083 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21084 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21085 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21086 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21087 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21088 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21089 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21090 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21091 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21092 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21093 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21094 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21095 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21096 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15165 MB total=81079 MB deficit_sync=-14909 MB recorded=51145 MB
|
| 21097 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21098 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21099 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21100 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21101 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21102 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21103 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21104 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21105 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21106 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21107 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21108 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21109 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21110 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21111 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21112 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21113 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21114 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21115 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21116 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21117 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21118 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21119 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21120 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21121 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21122 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21123 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21124 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21125 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21126 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21127 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21128 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21129 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21130 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21131 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21132 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21133 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21134 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21135 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21136 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21137 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21138 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21139 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21140 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21141 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21142 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21143 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21144 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21145 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21146 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21147 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21148 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21149 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21150 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21151 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21152 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21153 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21154 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21155 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21156 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21157 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21158 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21159 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21160 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21161 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21162 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21163 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21164 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21165 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21166 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21167 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21168 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21169 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21170 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21171 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21172 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21173 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21174 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21175 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21176 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21177 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21178 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21179 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21180 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21181 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21182 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21183 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21184 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21185 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21186 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21187 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21188 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21189 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21190 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21191 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21192 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21193 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21194 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21195 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21196 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21197 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21198 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21199 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21200 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21201 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21202 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21203 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21204 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21205 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21206 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21207 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21208 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21209 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21210 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21211 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21212 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21213 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21214 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21215 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21216 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21217 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21218 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21219 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21220 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21221 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21222 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21223 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21224 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21225 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21226 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21227 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21228 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21229 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21230 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21231 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21232 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21233 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21234 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21235 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21236 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21237 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21238 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21239 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21240 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21241 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21242 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21243 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21244 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21245 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21246 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21247 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21248 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21249 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21250 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21251 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21252 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21253 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21254 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21255 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21256 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51973 MB
|
| 21257 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21258 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=46524 MB
|
| 21259 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21260 |
+
|
| 21261 |
35%|███▌ | 7/20 [09:57<18:31, 85.50s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 21262 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x781d4b61d080: Actual Resident VRAM = 0 MB
|
| 21263 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780f14ef3440: Actual Resident VRAM = 160 MB
|
| 21264 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x780ebb35b580: Actual Resident VRAM = 25888 MB
|
| 21265 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x7811c40107c0: Actual Resident VRAM = 20000 MB
|
| 21266 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 46048 MB
|
| 21267 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21268 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21269 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21270 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21271 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21272 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21273 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21274 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21275 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21276 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21277 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21278 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21279 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21280 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21281 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21282 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21283 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21284 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21285 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21286 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21287 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21288 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21289 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21290 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21291 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21292 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21293 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21294 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21295 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21296 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21297 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21298 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21299 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21300 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21301 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21302 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21303 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21304 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21305 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21306 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21307 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21308 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21309 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21310 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21311 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21312 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21313 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21314 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21315 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21316 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21317 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21318 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21319 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21320 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21321 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21322 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21323 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21324 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21325 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21326 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21327 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21328 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21329 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21330 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21331 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21332 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21333 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21334 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21335 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21336 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21337 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21338 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21339 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21340 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21341 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21342 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21343 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21344 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21345 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21346 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21347 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21348 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21349 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21350 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21351 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21352 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21353 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21354 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21355 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21356 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21357 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21358 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21359 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21360 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21361 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21362 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21363 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21364 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21365 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21366 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21367 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21368 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21369 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21370 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21371 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21372 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21373 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21374 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21375 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21376 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21377 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21378 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21379 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21380 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21381 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21382 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21383 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21384 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21385 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21386 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21387 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21388 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21389 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21390 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21391 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21392 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21393 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21394 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21395 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21396 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21397 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21398 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21399 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21400 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21401 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21402 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21403 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21404 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21405 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21406 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21407 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21408 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21409 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21410 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21411 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21412 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21413 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21414 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21415 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21416 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21417 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21418 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21419 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21420 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21421 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21422 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21423 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21424 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21425 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21426 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21427 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21428 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21429 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21430 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21431 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21432 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21433 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21434 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21435 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21436 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21437 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21438 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21439 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21440 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21441 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21442 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21443 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21444 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21445 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21446 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21447 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21448 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21449 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21450 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21451 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21452 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21453 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21454 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21455 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21456 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21457 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21458 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21459 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21460 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21461 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21462 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21463 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21464 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21465 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21466 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21467 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21468 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21469 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21470 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21471 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21472 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21473 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21474 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21475 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21476 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21477 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21478 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21479 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21480 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21481 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21482 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 21483 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21484 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21485 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21486 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21487 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21488 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21489 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21490 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21491 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21492 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21493 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21494 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21495 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21496 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21497 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=15694 MB total=81079 MB deficit_sync=-15438 MB recorded=51145 MB
|
| 21498 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 21499 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21500 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21501 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21502 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21503 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21504 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 21505 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 21506 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|