diff --git a/.gitattributes b/.gitattributes index 64716abf3962aa07cb28eca6c7b5f86e0f2ef26b..a3f022eff87781e8ea8a27a031ceb983ea695449 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2939,3 +2939,48 @@ results/versatile_diffusion/subj01/956.png filter=lfs diff=lfs merge=lfs -text results/versatile_diffusion/subj01/948.png filter=lfs diff=lfs merge=lfs -text results/versatile_diffusion/subj01/954.png filter=lfs diff=lfs merge=lfs -text results/versatile_diffusion/subj01/946.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/952.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/955.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/951.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/962.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/959.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/973.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/957.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/970.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/977.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/976.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/972.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/969.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/974.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/975.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/978.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/961.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/953.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/966.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/965.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/968.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/964.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/963.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/960.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/971.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/981.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/979.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/967.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/958.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/96.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/99.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/980.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/0.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/10.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/1.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/5.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/7.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/6.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/11.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/98.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/9.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/8.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/97.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/12.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/2.png filter=lfs diff=lfs merge=lfs -text +results/versatile_diffusion/subj01/roi/3.png filter=lfs diff=lfs merge=lfs -text diff --git a/results/versatile_diffusion/subj01/951.png b/results/versatile_diffusion/subj01/951.png new file mode 100644 index 0000000000000000000000000000000000000000..b7f49ecc337f1eec7217f5a767e1965003d39c09 --- /dev/null +++ b/results/versatile_diffusion/subj01/951.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a276bff91f94f939c33f1608e51d3bd06f02747117ee107990187eef76429558 +size 195890 diff --git a/results/versatile_diffusion/subj01/952.png b/results/versatile_diffusion/subj01/952.png new file mode 100644 index 0000000000000000000000000000000000000000..707ef84c888c79235c23564d81bfba1066057085 --- /dev/null +++ b/results/versatile_diffusion/subj01/952.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66e9f74cfe01cec8c7c9ece3a8784f34d27b22e8248a4a478bc5bcc74fed7b74 +size 185365 diff --git a/results/versatile_diffusion/subj01/953.png b/results/versatile_diffusion/subj01/953.png new file mode 100644 index 0000000000000000000000000000000000000000..908f7857b1322244bbeffce8d59afa5c3758bdf5 --- /dev/null +++ b/results/versatile_diffusion/subj01/953.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b38cc8b20770b656fe37178811ba00dffe09ebe665ea29a91694f803cc2ebb +size 139437 diff --git a/results/versatile_diffusion/subj01/955.png b/results/versatile_diffusion/subj01/955.png new file mode 100644 index 0000000000000000000000000000000000000000..c7fe5031a423d65b6e20ba67a91c8cc1320fd373 --- /dev/null +++ b/results/versatile_diffusion/subj01/955.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea235d76e79fb734ee8b862393c048d1554ae8d053f3a9cbf0362f4518545c3 +size 249643 diff --git a/results/versatile_diffusion/subj01/957.png b/results/versatile_diffusion/subj01/957.png new file mode 100644 index 0000000000000000000000000000000000000000..308669833371a9fb943b6df2d3aa012bf54224be --- /dev/null +++ b/results/versatile_diffusion/subj01/957.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e922cff64da16de13e21e64f3c25941278d35588d9be4b8a81731f6aa71079 +size 207781 diff --git a/results/versatile_diffusion/subj01/958.png b/results/versatile_diffusion/subj01/958.png new file mode 100644 index 0000000000000000000000000000000000000000..36c1bd36e67c80605dd094811ffbdea56d8701b5 --- /dev/null +++ b/results/versatile_diffusion/subj01/958.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd66e3e796fb3d89076cf5871d3683dcb0cef334709b5ada16bb31188309f01 +size 187770 diff --git a/results/versatile_diffusion/subj01/959.png b/results/versatile_diffusion/subj01/959.png new file mode 100644 index 0000000000000000000000000000000000000000..86e87ae529b0f0c603e9e46ab46eeedd9263ea35 --- /dev/null +++ b/results/versatile_diffusion/subj01/959.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e42df5009a12127c1806f1776622ae4d058e902d6e0b3a3ad7053db62a5669 +size 232136 diff --git a/results/versatile_diffusion/subj01/96.png b/results/versatile_diffusion/subj01/96.png new file mode 100644 index 0000000000000000000000000000000000000000..9705a69fce3b18c35222babe4afbc27908d092b2 --- /dev/null +++ b/results/versatile_diffusion/subj01/96.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ec138e007b81043823e29d5bc6f02c8d75640087f72898b38bc8bf957be2d1 +size 224424 diff --git a/results/versatile_diffusion/subj01/960.png b/results/versatile_diffusion/subj01/960.png new file mode 100644 index 0000000000000000000000000000000000000000..28bf88660a9adc639e11255275ad82743d8ccc1f --- /dev/null +++ b/results/versatile_diffusion/subj01/960.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc44614c3ff43722cf66caed6442383a764125408a1943b87e611534cb6bd31 +size 279306 diff --git a/results/versatile_diffusion/subj01/961.png b/results/versatile_diffusion/subj01/961.png new file mode 100644 index 0000000000000000000000000000000000000000..eff4bd4272949f366c77433f58e68b2cb62db171 --- /dev/null +++ b/results/versatile_diffusion/subj01/961.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e752df40cdcd7e7b14f865cf89f6dd48209665fe7e2880389cfbcf17f6deeb77 +size 215603 diff --git a/results/versatile_diffusion/subj01/962.png b/results/versatile_diffusion/subj01/962.png new file mode 100644 index 0000000000000000000000000000000000000000..4da334de863911883cfecb24888cd7b0e1e045c8 --- /dev/null +++ b/results/versatile_diffusion/subj01/962.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544ef8d7e98eac3b2a0e0500168345e4927774bd8fb4ab3f71092ffc02dde850 +size 168770 diff --git a/results/versatile_diffusion/subj01/963.png b/results/versatile_diffusion/subj01/963.png new file mode 100644 index 0000000000000000000000000000000000000000..dbd571c92f64dbd67e68c77d0e2795a308badb4e --- /dev/null +++ b/results/versatile_diffusion/subj01/963.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5099a79dbd878dab61ae9ab059b5789b3e3f6a5ac7d75f27e82e6cd0d1b8b1c +size 208180 diff --git a/results/versatile_diffusion/subj01/964.png b/results/versatile_diffusion/subj01/964.png new file mode 100644 index 0000000000000000000000000000000000000000..f7d06c73178f154e4b298d62139de527d104359d --- /dev/null +++ b/results/versatile_diffusion/subj01/964.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77ca6a4fbcfc0aaf281180760046a612ed5b07b9dad75c69102c0d5047532687 +size 198625 diff --git a/results/versatile_diffusion/subj01/965.png b/results/versatile_diffusion/subj01/965.png new file mode 100644 index 0000000000000000000000000000000000000000..4c47d6b0a39d1f2bddcd4a56c5f2ee5b82c485c2 --- /dev/null +++ b/results/versatile_diffusion/subj01/965.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4492a6d242a432ff5ac00b0bcc1e79fc2c5e9391f563870a71a129d17bf72d4 +size 264873 diff --git a/results/versatile_diffusion/subj01/966.png b/results/versatile_diffusion/subj01/966.png new file mode 100644 index 0000000000000000000000000000000000000000..d775ed9726f3920af204a4be02ff99ae0364f544 --- /dev/null +++ b/results/versatile_diffusion/subj01/966.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63cbe22f74767bdd5a1ef52d415960a404c6a3f7a7b551451eed382e7b1d5812 +size 193869 diff --git a/results/versatile_diffusion/subj01/967.png b/results/versatile_diffusion/subj01/967.png new file mode 100644 index 0000000000000000000000000000000000000000..007f628b6f00eb7ed463131de1153a6e9424b68e --- /dev/null +++ b/results/versatile_diffusion/subj01/967.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb95ed959d2b1010e67be2265623abb779d327b5ed5815d2dcd727215ccd5dc +size 184271 diff --git a/results/versatile_diffusion/subj01/968.png b/results/versatile_diffusion/subj01/968.png new file mode 100644 index 0000000000000000000000000000000000000000..5bc6cda6571a6e388ab5052fdf829201a8372273 --- /dev/null +++ b/results/versatile_diffusion/subj01/968.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0f73300c482c0a48bf2fde120485b7715bd94d9e91ecef08ff9183f0ea9a15e +size 163685 diff --git a/results/versatile_diffusion/subj01/969.png b/results/versatile_diffusion/subj01/969.png new file mode 100644 index 0000000000000000000000000000000000000000..ddaad3fe5cf5e39d8f4d362625fcb8c6089af610 --- /dev/null +++ b/results/versatile_diffusion/subj01/969.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b56b9e0872df9ec48055b7330db8382dd4bc54dd7cc64d098f3925415598ee +size 194041 diff --git a/results/versatile_diffusion/subj01/97.png b/results/versatile_diffusion/subj01/97.png new file mode 100644 index 0000000000000000000000000000000000000000..2b4ffa3a5d6747172ee8604365a693c09b557b23 --- /dev/null +++ b/results/versatile_diffusion/subj01/97.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d124b709f7c62075af45321a321b95bbeb6bd257632a18bdd4640b424d6ce1 +size 226167 diff --git a/results/versatile_diffusion/subj01/970.png b/results/versatile_diffusion/subj01/970.png new file mode 100644 index 0000000000000000000000000000000000000000..69730021ed1eeec0f8e052fec1658218e4ef6cef --- /dev/null +++ b/results/versatile_diffusion/subj01/970.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b9a49344624450f1c185f3914228d1f0486035ae913114a1627e7a118bee22 +size 209946 diff --git a/results/versatile_diffusion/subj01/971.png b/results/versatile_diffusion/subj01/971.png new file mode 100644 index 0000000000000000000000000000000000000000..e72b47bc95369abe3fa905299b57bd267c4c5c24 --- /dev/null +++ b/results/versatile_diffusion/subj01/971.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a224f7cd8dd19cb49529a4b9fb53400eb822806e748bfb153d037f7cf2eea46 +size 284970 diff --git a/results/versatile_diffusion/subj01/972.png b/results/versatile_diffusion/subj01/972.png new file mode 100644 index 0000000000000000000000000000000000000000..ada629dc863ef7f5f8d8e1b352e14637613a7276 --- /dev/null +++ b/results/versatile_diffusion/subj01/972.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b19a719c14fa005c8d744d7a112dbbcb3f38d273afcedcc793d7fdac6bf802 +size 201682 diff --git a/results/versatile_diffusion/subj01/973.png b/results/versatile_diffusion/subj01/973.png new file mode 100644 index 0000000000000000000000000000000000000000..d7221b22ee1036a51f0f872470db04e58ad29234 --- /dev/null +++ b/results/versatile_diffusion/subj01/973.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cfba086b2a4041f7bbafd06f1bbd66cf6de5a5607dd274ecf667f635f495b1d +size 166508 diff --git a/results/versatile_diffusion/subj01/974.png b/results/versatile_diffusion/subj01/974.png new file mode 100644 index 0000000000000000000000000000000000000000..7f38f78a301ccf701cd72eb9597c342911a93d99 --- /dev/null +++ b/results/versatile_diffusion/subj01/974.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:830f19987d783b804d0df861370b75ffe607596069ec1ae877beb14c3ae72157 +size 242029 diff --git a/results/versatile_diffusion/subj01/975.png b/results/versatile_diffusion/subj01/975.png new file mode 100644 index 0000000000000000000000000000000000000000..f89c8c5ce1462d749d8f066ea0aedb9802adf80c --- /dev/null +++ b/results/versatile_diffusion/subj01/975.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ae0b1cce0b56f7303a0540c564bee9df35e381ec77409a0b2b40ace1eb821f +size 225876 diff --git a/results/versatile_diffusion/subj01/976.png b/results/versatile_diffusion/subj01/976.png new file mode 100644 index 0000000000000000000000000000000000000000..5ae7aafb6a2d798e73dbda48f9a8dbe55bc2e317 --- /dev/null +++ b/results/versatile_diffusion/subj01/976.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba24b7aa30bc5f9ff1e7b44acba307d30f0e4f165e2aa729ae374049deb0c22c +size 229046 diff --git a/results/versatile_diffusion/subj01/977.png b/results/versatile_diffusion/subj01/977.png new file mode 100644 index 0000000000000000000000000000000000000000..a117233bac309615f63659ee416b3398e01d1590 --- /dev/null +++ b/results/versatile_diffusion/subj01/977.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde29bcff9cc1d38cc55ea7f38ed9f44d55bfd07c036e5a6bb133b46aaceef4a +size 166355 diff --git a/results/versatile_diffusion/subj01/978.png b/results/versatile_diffusion/subj01/978.png new file mode 100644 index 0000000000000000000000000000000000000000..68cbaf082277c26f061ef346928909a68becabbd --- /dev/null +++ b/results/versatile_diffusion/subj01/978.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef050df1901835631d643f4d4a964c0d50a24cccffbd732a950e724431b8539d +size 169917 diff --git a/results/versatile_diffusion/subj01/979.png b/results/versatile_diffusion/subj01/979.png new file mode 100644 index 0000000000000000000000000000000000000000..751fe7e15b6ad37ff24f38987a517fad77889361 --- /dev/null +++ b/results/versatile_diffusion/subj01/979.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:255bbbb60eba158583607528b1336f6ba20b7dff24944f395078b42b243657c9 +size 242162 diff --git a/results/versatile_diffusion/subj01/98.png b/results/versatile_diffusion/subj01/98.png new file mode 100644 index 0000000000000000000000000000000000000000..c566f8bab1396deda52dd2b48f5ed74885196b58 --- /dev/null +++ b/results/versatile_diffusion/subj01/98.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bde1724856d5f7f7399f4d2da7187fcc11e72dad1fb3759b7e7d4623b2fc1594 +size 180294 diff --git a/results/versatile_diffusion/subj01/980.png b/results/versatile_diffusion/subj01/980.png new file mode 100644 index 0000000000000000000000000000000000000000..4ebaaf2dda259b1ac691116a552c123a640a0a4e --- /dev/null +++ b/results/versatile_diffusion/subj01/980.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4337b52af8f7f7e48ef3e909680c0841a47432c4194b0326f468fe14eabe6387 +size 280526 diff --git a/results/versatile_diffusion/subj01/981.png b/results/versatile_diffusion/subj01/981.png new file mode 100644 index 0000000000000000000000000000000000000000..91aed61ca26cae8bf25347d94d5d8c6fee878cb7 --- /dev/null +++ b/results/versatile_diffusion/subj01/981.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5f36dd6bc61be4e3072394e1ff58b8a7a559bc6853dd5f378d6b5f5209d5295 +size 175579 diff --git a/results/versatile_diffusion/subj01/99.png b/results/versatile_diffusion/subj01/99.png new file mode 100644 index 0000000000000000000000000000000000000000..e7010321c0a6c35bececb6fa8158b708021f6208 --- /dev/null +++ b/results/versatile_diffusion/subj01/99.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68beb2c299114e00365d06cce2c5a02b640e4d77731d14ed0796660bad574dcf +size 204558 diff --git a/results/versatile_diffusion/subj01/roi/0.png b/results/versatile_diffusion/subj01/roi/0.png new file mode 100644 index 0000000000000000000000000000000000000000..fbddc062af1011bb9238ce79dd5df72d90446fac --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24392e1b50eccbbe1e16c9e2d4f715ff58bf87b087ea72938d91d669587f2748 +size 206980 diff --git a/results/versatile_diffusion/subj01/roi/1.png b/results/versatile_diffusion/subj01/roi/1.png new file mode 100644 index 0000000000000000000000000000000000000000..f0174aad35523a9f53445f3f15f99a7e0e284340 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45d77380be8c7ce03a7180b6f0a069daaa639f210fa9741558abc1bb1bbd03de +size 168477 diff --git a/results/versatile_diffusion/subj01/roi/10.png b/results/versatile_diffusion/subj01/roi/10.png new file mode 100644 index 0000000000000000000000000000000000000000..7ce106530efeb29ca464cb3bc35e6fdc1ce4102f --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/10.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeefacce4476ae9835582b9539bb9b1954a5eaa16c4e5106f3d32f0a8fd892a9 +size 203544 diff --git a/results/versatile_diffusion/subj01/roi/11.png b/results/versatile_diffusion/subj01/roi/11.png new file mode 100644 index 0000000000000000000000000000000000000000..57ba5d0ae70a23bd401529640f6d9a5ee661bdb8 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/11.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4e951a75cd75bdd015555708e8a3bbb7f500d1441affa3512730f3bbe275fa3 +size 185748 diff --git a/results/versatile_diffusion/subj01/roi/12.png b/results/versatile_diffusion/subj01/roi/12.png new file mode 100644 index 0000000000000000000000000000000000000000..e224ba7134607af40af7adf8a8d5252c4c8f7363 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/12.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac09d06bc56767682a31c3c8706c2d3b3536d31f88546f04c2769eb77f8337c8 +size 149067 diff --git a/results/versatile_diffusion/subj01/roi/2.png b/results/versatile_diffusion/subj01/roi/2.png new file mode 100644 index 0000000000000000000000000000000000000000..81773b33560de6852c4c4d3bc142845951480f04 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d27ab8a45d9a8719ce0c338cd011c036649757c91a1db3a7a1cbabe7d27862 +size 155818 diff --git a/results/versatile_diffusion/subj01/roi/3.png b/results/versatile_diffusion/subj01/roi/3.png new file mode 100644 index 0000000000000000000000000000000000000000..a96bc45574e60463e81a93fac8e0a58f6f7b2a9a --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6fdadef7cdb35c9c4eb82e88a1fd2f0c346917d09d0c77c3100b95042c6101f +size 150536 diff --git a/results/versatile_diffusion/subj01/roi/5.png b/results/versatile_diffusion/subj01/roi/5.png new file mode 100644 index 0000000000000000000000000000000000000000..2fc45e1f79e7da8e8257bf62c29dd7be546e51bf --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e652ac2bee6b6a7891cc7e9cc50272656d18f6349fc93527657d4f0135020d +size 141394 diff --git a/results/versatile_diffusion/subj01/roi/6.png b/results/versatile_diffusion/subj01/roi/6.png new file mode 100644 index 0000000000000000000000000000000000000000..f53b25f3d33769e5890d5a381d26787189636cff --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65a11ba344d030f48dd07462cbac3b98fc0257044a3bd4211248ff33cce2bd0 +size 207102 diff --git a/results/versatile_diffusion/subj01/roi/7.png b/results/versatile_diffusion/subj01/roi/7.png new file mode 100644 index 0000000000000000000000000000000000000000..e34485617b73f88d30f994b4ddfbbf080d0403e6 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b547991c126290aaf36a45ebdd4cc9d1e576be9b86955e017675718a96e6206 +size 192549 diff --git a/results/versatile_diffusion/subj01/roi/8.png b/results/versatile_diffusion/subj01/roi/8.png new file mode 100644 index 0000000000000000000000000000000000000000..8aa6ebd0196f66caa1cbd0a6c4fd2a5a313fa22c --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c17f3bcc7e97d9ef5b2733c06850e6b9f4f2dba1e987774880f4b2f32431a99 +size 276679 diff --git a/results/versatile_diffusion/subj01/roi/9.png b/results/versatile_diffusion/subj01/roi/9.png new file mode 100644 index 0000000000000000000000000000000000000000..646a38c65c754b244a003c3752c49f7931c28a68 --- /dev/null +++ b/results/versatile_diffusion/subj01/roi/9.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b9bb072d328059b954a2bc79243783811d8b88859ec8f54700aa625dbc88c1 +size 278604 diff --git a/scripts/clipvision_extract_features.py b/scripts/clipvision_extract_features.py new file mode 100644 index 0000000000000000000000000000000000000000..1cbbbcaf977172d1690246689ee84319a4abb4e0 --- /dev/null +++ b/scripts/clipvision_extract_features.py @@ -0,0 +1,88 @@ +import sys +sys.path.append('versatile_diffusion') +import os +import PIL +from PIL import Image +import numpy as np + +import torch +from lib.cfg_helper import model_cfg_bank +from lib.model_zoo import get_model +from lib.experiments.sd_default import color_adjust, auto_merge_imlist +from torch.utils.data import DataLoader, Dataset + +from lib.model_zoo.vd import VD +from lib.cfg_holder import cfg_unique_holder as cfguh +from lib.cfg_helper import get_command_line_args, cfg_initiates, load_cfg_yaml +import torchvision.transforms as T + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] + +cfgm_name = 'vd_noema' + +pth = 'versatile_diffusion/pretrained/vd-four-flow-v1-0-fp16-deprecated.pth' +cfgm = model_cfg_bank()(cfgm_name) +net = get_model()(cfgm) +sd = torch.load(pth, map_location='cpu') +net.load_state_dict(sd, strict=False) + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +net.clip = net.clip.to(device) + +class batch_generator_external_images(Dataset): + + def __init__(self, data_path): + self.data_path = data_path + self.im = np.load(data_path).astype(np.uint8) + + + def __getitem__(self,idx): + img = Image.fromarray(self.im[idx]) + img = T.functional.resize(img,(512,512)) + img = T.functional.to_tensor(img).float() + #img = img/255 + img = img*2 - 1 + return img + + def __len__(self): + return len(self.im) + +batch_size=1 +image_path = 'data/processed_data/subj{:02d}/nsd_train_stim_sub{}.npy'.format(sub,sub) +train_images = batch_generator_external_images(data_path = image_path) + +image_path = 'data/processed_data/subj{:02d}/nsd_test_stim_sub{}.npy'.format(sub,sub) +test_images = batch_generator_external_images(data_path = image_path) + +trainloader = DataLoader(train_images,batch_size,shuffle=False) +testloader = DataLoader(test_images,batch_size,shuffle=False) + +num_embed, num_features, num_test, num_train = 257, 768, len(test_images), len(train_images) + +train_clip = np.zeros((num_train,num_embed,num_features)) +test_clip = np.zeros((num_test,num_embed,num_features)) + +with torch.no_grad(): + for i,cin in enumerate(testloader): + print(i) + #ctemp = cin*2 - 1 + c = net.clip_encode_vision(cin) + test_clip[i] = c[0].cpu().numpy() + + np.save('data/extracted_features/subj{:02d}/nsd_clipvision_test.npy'.format(sub),test_clip) + + for i,cin in enumerate(trainloader): + print(i) + #ctemp = cin*2 - 1 + c = net.clip_encode_vision(cin) + train_clip[i] = c[0].cpu().numpy() + np.save('data/extracted_features/subj{:02d}/nsd_clipvision_train.npy'.format(sub),train_clip) + + + + diff --git a/scripts/clipvision_regression.py b/scripts/clipvision_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..d18bf1a0b5dc2f6badf8f54aa0eef1cbbabf8c8d --- /dev/null +++ b/scripts/clipvision_regression.py @@ -0,0 +1,71 @@ +import sys +import numpy as np +import sklearn.linear_model as skl +import pickle +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] + +train_path = 'data/processed_data/subj{:02d}/nsd_train_fmriavg_nsdgeneral_sub{}.npy'.format(sub,sub) +train_fmri = np.load(train_path) +test_path = 'data/processed_data/subj{:02d}/nsd_test_fmriavg_nsdgeneral_sub{}.npy'.format(sub,sub) +test_fmri = np.load(test_path) + +## Preprocessing fMRI + +train_fmri = train_fmri/300 +test_fmri = test_fmri/300 + + +norm_mean_train = np.mean(train_fmri, axis=0) +norm_scale_train = np.std(train_fmri, axis=0, ddof=1) +train_fmri = (train_fmri - norm_mean_train) / norm_scale_train +test_fmri = (test_fmri - norm_mean_train) / norm_scale_train + +print(np.mean(train_fmri),np.std(train_fmri)) +print(np.mean(test_fmri),np.std(test_fmri)) + +print(np.max(train_fmri),np.min(train_fmri)) +print(np.max(test_fmri),np.min(test_fmri)) + +num_voxels, num_train, num_test = train_fmri.shape[1], len(train_fmri), len(test_fmri) + + +train_clip = np.load('data/extracted_features/subj{:02d}/nsd_clipvision_train.npy'.format(sub)) +test_clip = np.load('data/extracted_features/subj{:02d}/nsd_clipvision_test.npy'.format(sub)) + +#train_clip = train_clip[:,1:,:] +num_samples,num_embed,num_dim = train_clip.shape + +print("Training Regression") +reg_w = np.zeros((num_embed,num_dim,num_voxels)).astype(np.float32) +reg_b = np.zeros((num_embed,num_dim)).astype(np.float32) +pred_clip = np.zeros_like(test_clip) +for i in range(num_embed): + + + reg = skl.Ridge(alpha=60000, max_iter=50000, fit_intercept=True) + reg.fit(train_fmri, train_clip[:,i]) + reg_w[i] = reg.coef_ + reg_b[i] = reg.intercept_ + + pred_test_latent = reg.predict(test_fmri) + std_norm_test_latent = (pred_test_latent - np.mean(pred_test_latent,axis=0)) / np.std(pred_test_latent,axis=0) + pred_clip[:,i] = std_norm_test_latent * np.std(train_clip[:,i],axis=0) + np.mean(train_clip[:,i],axis=0) + + print(i,reg.score(test_fmri,test_clip[:,i])) + + +np.save('data/predicted_features/subj{:02d}/nsd_clipvision_predtest_nsdgeneral.npy'.format(sub),pred_clip) + +datadict = { + 'weight' : reg_w, + 'bias' : reg_b, + +} + +with open('data/regression_weights/subj{:02d}/clipvision_regression_weights.pkl'.format(sub),"wb") as f: + pickle.dump(datadict,f) diff --git a/scripts/eval_extract_features.py b/scripts/eval_extract_features.py new file mode 100644 index 0000000000000000000000000000000000000000..34aaf6848fb59e1b80f24ac67499f7ba778249e2 --- /dev/null +++ b/scripts/eval_extract_features.py @@ -0,0 +1,147 @@ +import os +import sys +import numpy as np +import h5py +import scipy.io as spio +import nibabel as nib + +import torch +import torchvision +import torchvision.models as tvmodels +import torchvision.transforms as transforms +from torch.utils.data import DataLoader, Dataset +import torchvision.transforms as T +from PIL import Image +import clip + +import skimage.io as sio +from skimage import data, img_as_float +from skimage.transform import resize as imresize +from skimage.metrics import structural_similarity as ssim +import scipy as sp + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub", help="Subject Number", default=1) +args = parser.parse_args() +sub = int(args.sub) +assert sub in [0, 1, 2, 5, 7] + +images_dir = 'data/nsddata_stimuli/test_images' +feats_dir = 'data/eval_features/test_images' + +if sub in [1, 2, 5, 7]: + feats_dir = f'data/eval_features/subj{sub:02d}' + images_dir = f'results/versatile_diffusion/subj{sub:02d}' + +if not os.path.exists(feats_dir): + os.makedirs(feats_dir) + +class batch_generator_external_images(Dataset): + def __init__(self, data_path='', prefix='', net_name='clip'): + self.data_path = data_path + self.prefix = prefix + self.net_name = net_name + + if self.net_name == 'clip': + self.normalize = transforms.Normalize( + mean=[0.48145466, 0.4578275, 0.40821073], + std=[0.26862954, 0.26130258, 0.27577711] + ) + else: + self.normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225] + ) + self.num_test = 982 + + def __getitem__(self, idx): + img = Image.open(f'{self.data_path}/{self.prefix}{idx}.png') + img = T.functional.resize(img, (224, 224)) + img = T.functional.to_tensor(img).float() + img = self.normalize(img) + return img + + def __len__(self): + return self.num_test + +# Set device +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +global feat_list +feat_list = [] + +def fn(module, inputs, outputs): + feat_list.append(outputs.cpu().numpy()) + +net_list = [ + ('inceptionv3', 'avgpool'), + ('clip', 'final'), + ('alexnet', 2), + ('alexnet', 5), + ('efficientnet', 'avgpool'), + ('swav', 'avgpool') +] + +batchsize = 64 + +for (net_name, layer) in net_list: + feat_list = [] + print(net_name, layer) + + dataset = batch_generator_external_images(data_path=images_dir, net_name=net_name, prefix='') + loader = DataLoader(dataset, batchsize, shuffle=False) + + if net_name == 'inceptionv3': + net = tvmodels.inception_v3(pretrained=True) + if layer == 'avgpool': + net.avgpool.register_forward_hook(fn) + elif layer == 'lastconv': + net.Mixed_7c.register_forward_hook(fn) + + elif net_name == 'alexnet': + net = tvmodels.alexnet(pretrained=True) + if layer == 2: + net.features[4].register_forward_hook(fn) + elif layer == 5: + net.features[11].register_forward_hook(fn) + elif layer == 7: + net.classifier[5].register_forward_hook(fn) + + elif net_name == 'clip': + model, _ = clip.load("ViT-L/14", device=device) + net = model.visual.to(torch.float32) + if layer == 7: + net.transformer.resblocks[7].register_forward_hook(fn) + elif layer == 12: + net.transformer.resblocks[12].register_forward_hook(fn) + elif layer == 'final': + net.register_forward_hook(fn) + + elif net_name == 'efficientnet': + net = tvmodels.efficientnet_b1(weights='IMAGENET1K_V1') + net.avgpool.register_forward_hook(fn) + + elif net_name == 'swav': + net = torch.hub.load('facebookresearch/swav:main', 'resnet50') + net.avgpool.register_forward_hook(fn) + + net.eval() + net = net.to(device) + + with torch.no_grad(): + for i, x in enumerate(loader): + print(i * batchsize) + x = x.to(device) + _ = net(x) + + if net_name == 'clip': + if layer == 7 or layer == 12: + feat_list = np.concatenate(feat_list, axis=1).transpose((1, 0, 2)) + else: + feat_list = np.concatenate(feat_list) + else: + feat_list = np.concatenate(feat_list) + + file_name = f'{feats_dir}/{net_name}_{layer}.npy' + np.save(file_name, feat_list) diff --git a/scripts/evaluate_reconstruction.py b/scripts/evaluate_reconstruction.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c3757977ed3253501cef21e3b29279813ae12b --- /dev/null +++ b/scripts/evaluate_reconstruction.py @@ -0,0 +1,93 @@ +import os +import sys +import numpy as np +import h5py +import scipy.io as spio +import nibabel as nib +import scipy as sp +from PIL import Image + + + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] + + +from scipy.stats import pearsonr,binom,linregress +import numpy as np +def pairwise_corr_all(ground_truth, predictions): + r = np.corrcoef(ground_truth, predictions)#cosine_similarity(ground_truth, predictions)# + r = r[:len(ground_truth), len(ground_truth):] # rows: groundtruth, columns: predicitons + #print(r.shape) + # congruent pairs are on diagonal + congruents = np.diag(r) + #print(congruents) + + # for each column (predicition) we should count the number of rows (groundtruth) that the value is lower than the congruent (e.g. success). + success = r < congruents + success_cnt = np.sum(success, 0) + + # note: diagonal of 'success' is always zero so we can discard it. That's why we divide by len-1 + perf = np.mean(success_cnt) / (len(ground_truth)-1) + p = 1 - binom.cdf(perf*len(ground_truth)*(len(ground_truth)-1), len(ground_truth)*(len(ground_truth)-1), 0.5) + + return perf, p + + +net_list = [ + ('inceptionv3','avgpool'), + ('clip','final'), + ('alexnet',2), + ('alexnet',5), + ('efficientnet','avgpool'), + ('swav','avgpool') + ] + +feats_dir = 'data/eval_features/subj{:02d}'.format(sub) +test_dir = 'data/eval_features/test_images' +num_test = 982 +distance_fn = sp.spatial.distance.correlation +pairwise_corrs = [] +for (net_name,layer) in net_list: + file_name = '{}/{}_{}.npy'.format(test_dir,net_name,layer) + gt_feat = np.load(file_name) + + file_name = '{}/{}_{}.npy'.format(feats_dir,net_name,layer) + eval_feat = np.load(file_name) + + gt_feat = gt_feat.reshape((len(gt_feat),-1)) + eval_feat = eval_feat.reshape((len(eval_feat),-1)) + + print(net_name,layer) + if net_name in ['efficientnet','swav']: + print('distance: ',np.array([distance_fn(gt_feat[i],eval_feat[i]) for i in range(num_test)]).mean()) + else: + pairwise_corrs.append(pairwise_corr_all(gt_feat[:num_test],eval_feat[:num_test])[0]) + print('pairwise corr: ',pairwise_corrs[-1]) + +from skimage.color import rgb2gray +from skimage.metrics import structural_similarity as ssim + +ssim_list = [] +pixcorr_list = [] +for i in range(982): + gen_image = Image.open('results/versatile_diffusion/subj{:02d}/{}.png'.format(sub,i)).resize((425,425)) + gt_image = Image.open('data/nsddata_stimuli/test_images/{}.png'.format(i)) + gen_image = np.array(gen_image)/255.0 + gt_image = np.array(gt_image)/255.0 + pixcorr_res = np.corrcoef(gt_image.reshape(1,-1), gen_image.reshape(1,-1))[0,1] + pixcorr_list.append(pixcorr_res) + gen_image = rgb2gray(gen_image) + gt_image = rgb2gray(gt_image) + ssim_res = ssim(gen_image, gt_image, multichannel=True, gaussian_weights=True, sigma=1.5, use_sample_covariance=False, data_range=1.0) + ssim_list.append(ssim_res) + +ssim_list = np.array(ssim_list) +pixcorr_list = np.array(pixcorr_list) +print('PixCorr: {}'.format(pixcorr_list.mean())) +print('SSIM: {}'.format(ssim_list.mean())) + diff --git a/scripts/roi_extract.py b/scripts/roi_extract.py new file mode 100644 index 0000000000000000000000000000000000000000..26cfe54f13d6c192666f8b1e6dac77ef2fda48e2 --- /dev/null +++ b/scripts/roi_extract.py @@ -0,0 +1,81 @@ +import os +import sys +import numpy as np +import h5py +import scipy.io as spio +import nibabel as nib + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] + + +roi_dir = 'data/nsddata/ppdata/subj{:02d}/func1pt8mm/roi/'.format(sub) +betas_dir = 'data/nsddata_betas/ppdata/subj{:02d}/func1pt8mm/betas_fithrf_GLMdenoise_RR/'.format(sub) +res_roi_dir = 'data/processed_data/subj{:02d}/roi/'.format(sub) +if not os.path.exists(res_roi_dir): + os.makedirs(res_roi_dir) + +nsdgeneral_mask_filename = 'nsdgeneral.nii.gz' +nsdgeneral_mask = nib.load(roi_dir+nsdgeneral_mask_filename).get_fdata() +nsdgeneral_mask[nsdgeneral_mask<0] = 0 +num_voxel = nsdgeneral_mask[nsdgeneral_mask>0].shape[0] +print(f'NSD General : {num_voxel}') + + +mask_files = [ + 'floc-faces.nii.gz', + 'floc-words.nii.gz', + 'floc-places.nii.gz', + 'floc-bodies.nii.gz' + ] + + + +for mfile in mask_files: + roi_mask = nib.load(roi_dir+mfile).get_fdata() + np.save('data/processed_data/subj{:02d}/roi/{}.npy'.format(sub,mfile[:-7]), roi_mask[nsdgeneral_mask>0]) + + +roi_mask = nib.load(roi_dir+mask_files[0]).get_fdata() +v1 = np.zeros_like(nsdgeneral_mask) +v2 = np.zeros_like(nsdgeneral_mask) +v3 = np.zeros_like(nsdgeneral_mask) +v4 = np.zeros_like(nsdgeneral_mask) + +v1[roi_mask==1] = 1 +v1[roi_mask==2] = 1 +v2[roi_mask==3] = 1 +v2[roi_mask==4] = 1 +v3[roi_mask==5] = 1 +v3[roi_mask==6] = 1 +v4[roi_mask==7] = 1 + +np.save('data/processed_data/subj{:02d}/roi/V1.npy'.format(sub), v1[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/V2.npy'.format(sub), v2[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/V3.npy'.format(sub), v3[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/V4.npy'.format(sub), v4[nsdgeneral_mask>0]) + + +roi_mask = nib.load(roi_dir+"prf-eccrois.nii.gz").get_fdata() +ecc05 = np.zeros_like(nsdgeneral_mask) +ecc10 = np.zeros_like(nsdgeneral_mask) +ecc20 = np.zeros_like(nsdgeneral_mask) +ecc40 = np.zeros_like(nsdgeneral_mask) +ecc40p = np.zeros_like(nsdgeneral_mask) + +ecc05[roi_mask==1] = 1 +ecc10[roi_mask==2] = 1 +ecc20[roi_mask==3] = 1 +ecc40[roi_mask==4] = 1 +ecc40p[roi_mask==5] = 1 + +np.save('data/processed_data/subj{:02d}/roi/ecc05.npy'.format(sub), ecc05[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/ecc10.npy'.format(sub), ecc10[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/ecc20.npy'.format(sub), ecc20[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/ecc40.npy'.format(sub), ecc40[nsdgeneral_mask>0]) +np.save('data/processed_data/subj{:02d}/roi/ecc40p.npy'.format(sub), ecc40p[nsdgeneral_mask>0]) + diff --git a/scripts/roi_generate_features.py b/scripts/roi_generate_features.py new file mode 100644 index 0000000000000000000000000000000000000000..a235761eb1ba59bc3e9fbb7fe75e5a2439f3351a --- /dev/null +++ b/scripts/roi_generate_features.py @@ -0,0 +1,85 @@ +import numpy as np +import pickle + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] + +# Load ROI Masks + +with open('data/regression_weights/subj{:02d}/vdvae_regression_weights.pkl'.format(sub),"rb") as f: + datadict = pickle.load(f) + reg_w = datadict['weight'] + reg_b = datadict['bias'] + +roi_dir = 'data/processed_data/subj{:02d}/roi'.format(sub) +num_rois = 13 +roi_act=np.zeros((num_rois,reg_w.shape[1])).astype(np.float32) +roi_act[0] = np.load("{}/floc-faces.npy".format(roi_dir)) +roi_act[1] = np.load("{}/floc-words.npy".format(roi_dir)) +roi_act[2] = np.load("{}/floc-places.npy".format(roi_dir)) +roi_act[3] = np.load("{}/floc-bodies.npy".format(roi_dir)) +roi_act[4] = np.load("{}/V1.npy".format(roi_dir)) +roi_act[5] = np.load("{}/V2.npy".format(roi_dir)) +roi_act[6] = np.load("{}/V3.npy".format(roi_dir)) +roi_act[7] = np.load("{}/V4.npy".format(roi_dir)) +roi_act[8] = np.load("{}/ecc05.npy".format(roi_dir)) +roi_act[9] = np.load("{}/ecc10.npy".format(roi_dir)) +roi_act[10] = np.load("{}/ecc20.npy".format(roi_dir)) +roi_act[11] = np.load("{}/ecc40.npy".format(roi_dir)) +roi_act[12] = np.load("{}/ecc40p.npy".format(roi_dir)) + + +roi_act[roi_act>0]=1 +roi_act[roi_act<0]=0 + +# Generate VDVAE Features + +nsd_features = np.load('data/extracted_features/subj{:02d}/nsd_vdvae_features_31l.npz'.format(sub)) +train_latents = nsd_features['train_latents'] + +pred_vae = (roi_act @ reg_w.T) +pred_vae = pred_vae / (np.linalg.norm(pred_vae,axis=1).reshape((num_rois,1)) + 1e-8) +pred_vae = pred_vae * 50 + reg_b + +pred_vae = (pred_vae - np.mean(pred_vae,axis=0)) / np.std(pred_vae,axis=0) + +pred_vae = pred_vae * np.std(train_latents,axis=0) + np.mean(train_latents,axis=0) +pred_vae = pred_vae / np.linalg.norm(pred_vae,axis=1).reshape((num_rois,1)) +pred_vae = pred_vae * 80 +np.save('data/predicted_features/subj{:02d}/nsd_vdvae_nsdgeneral_roi_sub{}_31l_alpha50k.npy'.format(sub,sub),pred_vae) + +# Generate CLIP-Text Features + +with open('data/regression_weights/subj{:02d}/cliptext_regression_weights.pkl'.format(sub),"rb") as f: + datadict = pickle.load(f) + reg_w = datadict['weight'] + reg_b = datadict['bias'] + +pred_clipt = np.zeros((num_rois,reg_w.shape[0],reg_w.shape[1])).astype(np.float32) +for j in range(reg_w.shape[0]): + pred_clipt[:,j] = (roi_act @ reg_w[j].T) + +pred_clipt = pred_clipt / (np.linalg.norm(pred_clipt,axis=(1,2)).reshape((num_rois,1,1)) + 1e-8) +pred_clipt = pred_clipt * 9 + reg_b + +np.save('data/predicted_features/subj{:02d}/nsd_cliptext_roi_nsdgeneral.npy'.format(sub),pred_clipt) + +# Generate CLIP-Vision Features + +with open('data/regression_weights/subj{:02d}/clipvision_regression_weights.pkl'.format(sub),"rb") as f: + datadict = pickle.load(f) + reg_w = datadict['weight'] + reg_b = datadict['bias'] + +pred_clipv = np.zeros((num_rois,reg_w.shape[0],reg_w.shape[1])).astype(np.float32) +for j in range(reg_w.shape[0]): + pred_clipv[:,j] = (roi_act @ reg_w[j].T) + +pred_clipv = pred_clipv / (np.linalg.norm(pred_clipv,axis=(1,2)).reshape((num_rois,1,1)) + 1e-8) +pred_clipv = pred_clipv * 15 + reg_b + +np.save('data/predicted_features/subj{:02d}/nsd_clipvision_roi_nsdgeneral.npy'.format(sub),pred_clipv) diff --git a/scripts/roi_vdvae_reconstruct.py b/scripts/roi_vdvae_reconstruct.py new file mode 100644 index 0000000000000000000000000000000000000000..1b74de892f5b81caae52653ab78c913db0c812d3 --- /dev/null +++ b/scripts/roi_vdvae_reconstruct.py @@ -0,0 +1,142 @@ +import sys +sys.path.append('vdvae') +import torch +import numpy as np +#from mpi4py import MPI +import socket +import argparse +import os +import json +import subprocess +from hps import Hyperparams, parse_args_and_update_hparams, add_vae_arguments +from utils import (logger, + local_mpi_rank, + mpi_size, + maybe_download, + mpi_rank) +from data import mkdir_p +from contextlib import contextmanager +import torch.distributed as dist +#from apex.optimizers import FusedAdam as AdamW +from vae import VAE +from torch.nn.parallel.distributed import DistributedDataParallel +from train_helpers import restore_params +from image_utils import * +from model_utils import * +from torch.utils.data import DataLoader, Dataset +from PIL import Image +import torchvision.transforms as T +import pickle + +import argparse +parser = argparse.ArgumentParser(description='Argument Parser') +parser.add_argument("-sub", "--sub",help="Subject Number",default=1) +parser.add_argument("-bs", "--bs",help="Batch Size",default=30) +args = parser.parse_args() +sub=int(args.sub) +assert sub in [1,2,5,7] +batch_size=int(args.bs) + +print('Libs imported') + +H = {'image_size': 64, 'image_channels': 3,'seed': 0, 'port': 29500, 'save_dir': './saved_models/test', 'data_root': './', 'desc': 'test', 'hparam_sets': 'imagenet64', 'restore_path': 'imagenet64-iter-1600000-model.th', 'restore_ema_path': 'vdvae/model/imagenet64-iter-1600000-model-ema.th', 'restore_log_path': 'imagenet64-iter-1600000-log.jsonl', 'restore_optimizer_path': 'imagenet64-iter-1600000-opt.th', 'dataset': 'imagenet64', 'ema_rate': 0.999, 'enc_blocks': '64x11,64d2,32x20,32d2,16x9,16d2,8x8,8d2,4x7,4d4,1x5', 'dec_blocks': '1x2,4m1,4x3,8m4,8x7,16m8,16x15,32m16,32x31,64m32,64x12', 'zdim': 16, 'width': 512, 'custom_width_str': '', 'bottleneck_multiple': 0.25, 'no_bias_above': 64, 'scale_encblock': False, 'test_eval': True, 'warmup_iters': 100, 'num_mixtures': 10, 'grad_clip': 220.0, 'skip_threshold': 380.0, 'lr': 0.00015, 'lr_prior': 0.00015, 'wd': 0.01, 'wd_prior': 0.0, 'num_epochs': 10000, 'n_batch': 4, 'adam_beta1': 0.9, 'adam_beta2': 0.9, 'temperature': 1.0, 'iters_per_ckpt': 25000, 'iters_per_print': 1000, 'iters_per_save': 10000, 'iters_per_images': 10000, 'epochs_per_eval': 1, 'epochs_per_probe': None, 'epochs_per_eval_save': 1, 'num_images_visualize': 8, 'num_variables_visualize': 6, 'num_temperatures_visualize': 3, 'mpi_size': 1, 'local_rank': 0, 'rank': 0, 'logdir': './saved_models/test/log'} +class dotdict(dict): + """dot.notation access to dictionary attributes""" + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ +H = dotdict(H) + +H, preprocess_fn = set_up_data(H) + +print('Models is Loading') +ema_vae = load_vaes(H) + + +class batch_generator_external_images(Dataset): + + def __init__(self, data_path): + self.data_path = data_path + self.im = np.load(data_path).astype(np.uint8) + + + def __getitem__(self,idx): + img = Image.fromarray(self.im[idx]) + img = T.functional.resize(img,(64,64)) + img = torch.tensor(np.array(img)).float() + #img = img/255 + #img = img*2 - 1 + return img + + def __len__(self): + return len(self.im) + + + +image_path = 'data/processed_data/subj{:02d}/nsd_test_stim_sub{}.npy'.format(sub,sub) +test_images = batch_generator_external_images(data_path = image_path) +testloader = DataLoader(test_images,batch_size,shuffle=False) + +test_latents = [] +for i,x in enumerate(testloader): + data_input, target = preprocess_fn(x) + with torch.no_grad(): + print(i*batch_size) + activations = ema_vae.encoder.forward(data_input) + px_z, stats = ema_vae.decoder.forward(activations, get_latents=True) + #recons = ema_vae.decoder.out_net.sample(px_z) + batch_latent = [] + for i in range(31): + batch_latent.append(stats[i]['z'].cpu().numpy().reshape(len(data_input),-1)) + test_latents.append(np.hstack(batch_latent)) + #stats_all.append(stats) + #imshow(imgrid(recons, cols=batch_size,pad=20)) + #imshow(imgrid(test_images[i*batch_size : (i+1)*batch_size], cols=batch_size,pad=20)) +test_latents = np.concatenate(test_latents) + +pred_latents = np.load('data/predicted_features/subj{:02d}/nsd_vdvae_nsdgeneral_roi_sub{}_31l_alpha50k.npy'.format(sub,sub)) +ref_latent = stats + +# Transfor latents from flattened representation to hierarchical +def latent_transformation(latents, ref): + layer_dims = np.array([2**4,2**4,2**8,2**8,2**8,2**8,2**10,2**10,2**10,2**10,2**10,2**10,2**10,2**10,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**12,2**14]) + transformed_latents = [] + for i in range(31): + t_lat = latents[:,layer_dims[:i].sum():layer_dims[:i+1].sum()] + #std_norm_test_latent = (t_lat - np.mean(t_lat,axis=0)) / np.std(t_lat,axis=0) + #renorm_test_latent = std_norm_test_latent * np.std(kamitani_latents[i][num_test:].reshape(num_train,-1),axis=0) + np.mean(kamitani_latents[i][num_test:].reshape(num_train,-1),axis=0) + c,h,w=ref[i]['z'].shape[1:] + transformed_latents.append(t_lat.reshape(len(latents),c,h,w)) + return transformed_latents + +idx = range(13) +input_latent = latent_transformation(pred_latents[idx],ref_latent) + + +def sample_from_hier_latents(latents,sample_ids): + sample_ids = [id for id in sample_ids if id= 1 + N, H, W, C = imarray.shape + rows = N // cols + int(N % cols != 0) + batch_pad = rows * cols - N + assert batch_pad >= 0 + post_pad = [batch_pad, pad, pad, 0] + pad_arg = [[0, p] for p in post_pad] + imarray = np.pad(imarray, pad_arg, 'constant', constant_values=padval) + H += pad + W += pad + grid = (imarray + .reshape(rows, cols, H, W, C) + .transpose(0, 2, 1, 3, 4) + .reshape(rows*H, cols*W, C)) + if pad: + grid = grid[:-pad, :-pad] + return grid + +def interleave(*args): + """Interleaves input arrays of the same shape along the batch axis.""" + if not args: + raise ValueError('At least one argument is required.') + a0 = args[0] + if any(a.shape != a0.shape for a in args): + raise ValueError('All inputs must have the same shape.') + if not a0.shape: + raise ValueError('Inputs must have at least one axis.') + out = np.transpose(args, [1, 0] + list(range(2, len(a0.shape) + 1))) + out = out.reshape(-1, *a0.shape[1:]) + return out + +def imshow(a, format='png', jpeg_fallback=True): + """Displays an image in the given format.""" + a = a.astype(np.uint8) + data = io.BytesIO() + PIL.Image.fromarray(a).save(data, format) + im_data = data.getvalue() + try: + disp = IPython.display.display(IPython.display.Image(im_data)) + except IOError: + if jpeg_fallback and format != 'jpeg': + print ('Warning: image was too large to display in format "{}"; ' + 'trying jpeg instead.').format(format) + return imshow(a, format='jpeg') + else: + raise + return disp + +def image_to_uint8(x): + """Converts [-1, 1] float array to [0, 255] uint8.""" + x = np.asarray(x) + x = (256. / 2.) * (x + 1.) + x = np.clip(x, 0, 255) + x = x.astype(np.uint8) + return x diff --git a/vdvae/model/imagenet64-iter-1600000-log.jsonl b/vdvae/model/imagenet64-iter-1600000-log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fc452c1720cbcc70932371bd860e25d697c04029 --- /dev/null +++ b/vdvae/model/imagenet64-iter-1600000-log.jsonl @@ -0,0 +1,473 @@ +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "adam_beta1", "value": 0.9} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "adam_beta2", "value": 0.9} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "bottleneck_multiple", "value": 0.25} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "custom_width_str", "value": ""} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "data_root", "value": "/root/data"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "dataset", "value": "imagenet64"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "dec_blocks", "value": "1x2,4m1,4x3,8m4,8x7,16m8,16x15,32m16,32x31,64m32,64x12"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "desc", "value": "oct11-i64-re1-lr05"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "ema_rate", "value": 0.999} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "enc_blocks", "value": "64x11,64d2,32x20,32d2,16x9,16d2,8x8,8d2,4x7,4d4,1x5"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "epochs_per_eval", "value": 1} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "epochs_per_eval_save", "value": 1} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "epochs_per_probe", "value": null} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "grad_clip", "value": 220.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "gs_backup", "value": "gs://rewon/vaes"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "hparam_sets", "value": "i64"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "iters_per_backup", "value": 10000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "iters_per_ckpt", "value": 25000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "iters_per_images", "value": 10000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "iters_per_print", "value": 1000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "iters_per_save", "value": 10000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "local_rank", "value": 0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "logdir", "value": "/root/vaes/oct11-i64-re1-lr05/log"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "lr", "value": 5e-05} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "lr_prior", "value": 0.00015} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "mpi_size", "value": 32} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "n_batch", "value": 4} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "no_bias_above", "value": 64} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "num_epochs", "value": 10000} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "num_images_visualize", "value": 8} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "num_mixtures", "value": 10} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "num_temperatures_visualize", "value": 3} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "num_variables_visualize", "value": 6} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "port", "value": 29500} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "prior_warmup_iters", "value": 1.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "rank", "value": 0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "restore_ema_path", "value": "gs://rewon/vaes/oct11-i64/latest-model-ema.th"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "restore_log_path", "value": "gs://rewon/vaes/oct11-i64/latest-log.jsonl"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "restore_optimizer_path", "value": "gs://rewon/vaes/oct11-i64/latest-opt.th"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "restore_path", "value": "gs://rewon/vaes/oct11-i64/latest-model.th"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "save_dir", "value": "/root/vaes/oct11-i64-re1-lr05"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "save_dir_gcp", "value": "gs://rewon/vaes/oct11-i64-re1-lr05"} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "scale_encblock", "value": false} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "seed", "value": 0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "skip_threshold", "value": 380.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "temperature", "value": 1.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "test_eval", "value": false} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "warmup_iters", "value": 1.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "wd", "value": 0.01} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "wd_prior", "value": 0.0} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "width", "value": 512} +{"time": "Wed Nov 11 21:13:38 2020", "type": "hparam", "key": "zdim", "value": 16} +{"time": "Wed Nov 11 21:13:38 2020", "message": "training model oct11-i64-re1-lr05 on imagenet64"} +{"time": "Wed Nov 11 21:14:01 2020", "message": "Restoring vae from gs://rewon/vaes/oct11-i64/latest-model.th"} +{"time": "Wed Nov 11 21:14:12 2020", "message": "Restoring swa vae from gs://rewon/vaes/oct11-i64/latest-model-ema.th"} +{"time": "Wed Nov 11 21:14:25 2020", "total_params": 125005860, "readable": "125,005,860"} +{"time": "Wed Nov 11 21:14:44 2020", "message": "starting at epoch 132 iterate 1320000 eval loss 2.3909541093386135"} +{"time": "Wed Nov 11 21:14:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320000, "distortion": 1.445999264717102, "distortion_nans": 0.0, "elbo": 2.4574265480041504, "elbo_filtered": 2.4574265480041504, "rate": 1.0114271640777588, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 7.248043775558472, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:14:53 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320000.png"} +{"time": "Wed Nov 11 21:14:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320001, "distortion": 1.4456197023391724, "distortion_nans": 0.0, "elbo": 2.4434181451797485, "elbo_filtered": 2.4434181451797485, "rate": 0.997798353433609, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 2.5710151195526123, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:14:58 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320001.png"} +{"time": "Wed Nov 11 21:15:09 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320008, "distortion": 1.413430306646559, "distortion_nans": 0.0, "elbo": 2.3972657786475287, "elbo_filtered": 2.3972657786475287, "rate": 0.9838354521327548, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.5174002647399902, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:15:10 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320008.png"} +{"time": "Wed Nov 11 21:15:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320016, "distortion": 1.4091368983773624, "distortion_nans": 0.0, "elbo": 2.4029655316296745, "elbo_filtered": 2.4029655316296745, "rate": 0.9938286472769344, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.2357289791107178, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:15:23 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320016.png"} +{"time": "Wed Nov 11 21:15:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320032, "distortion": 1.3927589076938052, "distortion_nans": 0.0, "elbo": 2.3946841485572583, "elbo_filtered": 2.3946841485572583, "rate": 1.0019252661502722, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.40993332862854, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:15:46 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320032.png"} +{"time": "Wed Nov 11 21:16:30 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320064, "distortion": 1.379488048186669, "distortion_nans": 0.0, "elbo": 2.3907551802121674, "elbo_filtered": 2.3907551802121674, "rate": 1.0112671439464276, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.463927984237671, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:16:31 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320064.png"} +{"time": "Wed Nov 11 21:17:57 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320128, "distortion": 1.3717271542364313, "distortion_nans": 0.0, "elbo": 2.390639615613361, "elbo_filtered": 2.390639615613361, "rate": 1.0189124710800113, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3674860000610352, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:17:58 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320128.png"} +{"time": "Wed Nov 11 21:20:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320256, "distortion": 1.3663373730989745, "distortion_nans": 0.0, "elbo": 2.392330786597404, "elbo_filtered": 2.392330786597404, "rate": 1.025993404917216, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.223928451538086, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:20:47 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320256.png"} +{"time": "Wed Nov 11 21:26:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1320512, "distortion": 1.3550183703327736, "distortion_nans": 0.0, "elbo": 2.3887336198349445, "elbo_filtered": 2.3887336198349445, "rate": 1.033715254265895, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3158071041107178, "grad_norm": 24.829143524169922} +{"time": "Wed Nov 11 21:26:28 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1320512.png"} +{"time": "Wed Nov 11 21:37:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1321000, "distortion": 1.3473977916240691, "distortion_nans": 0.0, "elbo": 2.38857044172287, "elbo_filtered": 2.38857044172287, "rate": 1.041172653257847, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3320198609828948, "grad_norm": 26.993892669677734} +{"time": "Wed Nov 11 21:37:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1321024, "distortion": 1.3460593459606172, "distortion_nans": 0.0, "elbo": 2.38874130821228, "elbo_filtered": 2.38874130821228, "rate": 1.042681964635849, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3287182281017302, "grad_norm": 26.993892669677734} +{"time": "Wed Nov 11 21:37:54 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1321024.png"} +{"time": "Wed Nov 11 21:59:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1322000, "distortion": 1.3326473542451858, "distortion_nans": 0.0, "elbo": 2.3921887576580048, "elbo_filtered": 2.3921887576580048, "rate": 1.0595414137244226, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3140434329509736, "grad_norm": 25.44363784790039} +{"time": "Wed Nov 11 22:00:23 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1322048, "distortion": 1.3327344779968262, "distortion_nans": 0.0, "elbo": 2.3929132273197173, "elbo_filtered": 2.3929132273197173, "rate": 1.0601787588000298, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3169792516231538, "grad_norm": 25.44363784790039} +{"time": "Wed Nov 11 22:00:24 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1322048.png"} +{"time": "Wed Nov 11 22:21:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1323000, "distortion": 1.327664522409439, "distortion_nans": 0.0, "elbo": 2.391411930322647, "elbo_filtered": 2.391411930322647, "rate": 1.063747417330742, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3163880586624146, "grad_norm": 18.708385467529297} +{"time": "Wed Nov 11 22:43:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1324000, "distortion": 1.3203397040367126, "distortion_nans": 0.0, "elbo": 2.388113047361374, "elbo_filtered": 2.388113047361374, "rate": 1.0677733450531959, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3229240109920501, "grad_norm": 17.526649475097656} +{"time": "Wed Nov 11 22:45:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1324096, "distortion": 1.3187181377410888, "distortion_nans": 0.0, "elbo": 2.3871472980976103, "elbo_filtered": 2.3871472980976103, "rate": 1.0684291600584983, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3235159957408904, "grad_norm": 17.526649475097656} +{"time": "Wed Nov 11 22:45:30 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1324096.png"} +{"time": "Wed Nov 11 23:05:24 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1325000, "distortion": 1.316500957608223, "distortion_nans": 0.0, "elbo": 2.3893657047748564, "elbo_filtered": 2.3893657047748564, "rate": 1.0728647494316101, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3152212438583375, "grad_norm": 18.99942398071289} +{"time": "Wed Nov 11 23:27:22 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1326000, "distortion": 1.3145045186281203, "distortion_nans": 0.0, "elbo": 2.3853585493564604, "elbo_filtered": 2.3853585493564604, "rate": 1.0708540303707124, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176168694496155, "grad_norm": 19.138408660888672} +{"time": "Wed Nov 11 23:49:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1327000, "distortion": 1.313844377040863, "distortion_nans": 0.0, "elbo": 2.3867756128311157, "elbo_filtered": 2.3867756128311157, "rate": 1.0729312378168105, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3260120141506195, "grad_norm": 20.209964752197266} +{"time": "Thu Nov 12 00:11:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1328000, "distortion": 1.3147782752513886, "distortion_nans": 0.0, "elbo": 2.388324030160904, "elbo_filtered": 2.388324030160904, "rate": 1.073545749604702, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318836389541626, "grad_norm": 19.149518966674805} +{"time": "Thu Nov 12 00:15:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1328192, "distortion": 1.3139881466627121, "distortion_nans": 0.0, "elbo": 2.3880273895263673, "elbo_filtered": 2.3880273895263673, "rate": 1.0740392386317252, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3152216053009034, "grad_norm": 19.149518966674805} +{"time": "Thu Nov 12 00:15:42 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1328192.png"} +{"time": "Thu Nov 12 00:33:22 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 132, "step": 1329000, "distortion": 1.3090371681451798, "distortion_nans": 0.0, "elbo": 2.385475355386734, "elbo_filtered": 2.385475355386734, "rate": 1.0764381934404372, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3100676107406617, "grad_norm": 19.390241622924805} +{"time": "Thu Nov 12 00:54:48 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 132, "step": 1329970, "n_batches": 39, "filtered_elbo": 2.386846273373335, "distortion": 1.313077226663247, "elbo": 2.386846273373335, "rate": 1.0737690681066268} +{"time": "Thu Nov 12 00:55:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 133, "step": 1330000, "distortion": 1.309757377386093, "distortion_nans": 0.0, "elbo": 2.385722698688507, "elbo_filtered": 2.385722698688507, "rate": 1.0759653251767158, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3180113849639892, "grad_norm": 18.971315383911133} +{"time": "Thu Nov 12 00:55:27 2020", "message": "Saving model@ 1330000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 00:55:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1330000, "distortion": 1.3096617696285249, "distortion_nans": 0.0, "elbo": 2.3855494990348816, "elbo_filtered": 2.3855494990348816, "rate": 1.07588773304224, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.31812255525589, "grad_norm": 18.971315383911133} +{"time": "Thu Nov 12 00:55:35 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1330000.png"} +{"time": "Thu Nov 12 01:17:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1331000, "distortion": 1.3039979158639907, "distortion_nans": 0.0, "elbo": 2.3842448103427887, "elbo_filtered": 2.3842448103427887, "rate": 1.0802468958497047, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3276605627536773, "grad_norm": 22.192724227905273} +{"time": "Thu Nov 12 01:39:32 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1332000, "distortion": 1.3067725878953933, "distortion_nans": 0.0, "elbo": 2.3871798202991488, "elbo_filtered": 2.3871798202991488, "rate": 1.080407234430313, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3071586058139801, "grad_norm": 21.929092407226562} +{"time": "Thu Nov 12 02:01:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1333000, "distortion": 1.3011728899478912, "distortion_nans": 0.0, "elbo": 2.3819777352809908, "elbo_filtered": 2.3819777352809908, "rate": 1.0808048527240752, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3242859296798706, "grad_norm": 18.41106414794922} +{"time": "Thu Nov 12 02:23:48 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1334000, "distortion": 1.3051445759534837, "distortion_nans": 0.0, "elbo": 2.387447967529297, "elbo_filtered": 2.387447967529297, "rate": 1.0823033927083014, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3291694850921631, "grad_norm": 22.12964630126953} +{"time": "Thu Nov 12 02:45:49 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1335000, "distortion": 1.3055463256835937, "distortion_nans": 0.0, "elbo": 2.3869473433494566, "elbo_filtered": 2.3869473433494566, "rate": 1.0814010164737702, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.319949670791626, "grad_norm": 27.443634033203125} +{"time": "Thu Nov 12 03:07:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1336000, "distortion": 1.3045781100988387, "distortion_nans": 0.0, "elbo": 2.387768555164337, "elbo_filtered": 2.387768555164337, "rate": 1.0831904408931732, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.320764415025711, "grad_norm": 22.000080108642578} +{"time": "Thu Nov 12 03:29:58 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1337000, "distortion": 1.3045844731330871, "distortion_nans": 0.0, "elbo": 2.3865578258037567, "elbo_filtered": 2.3865578258037567, "rate": 1.0819733523130417, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3268116979598998, "grad_norm": 22.1534481048584} +{"time": "Thu Nov 12 03:52:07 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1338000, "distortion": 1.3064334908723831, "distortion_nans": 0.0, "elbo": 2.39144122338295, "elbo_filtered": 2.39144122338295, "rate": 1.0850077353715897, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3277591803073883, "grad_norm": 22.378437042236328} +{"time": "Thu Nov 12 04:14:08 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 133, "step": 1339000, "distortion": 1.304289416193962, "distortion_nans": 0.0, "elbo": 2.3866431653499602, "elbo_filtered": 2.3866431653499602, "rate": 1.082353764772415, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.319819466114044, "grad_norm": 18.97158432006836} +{"time": "Thu Nov 12 04:34:53 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 133, "step": 1339940, "n_batches": 39, "filtered_elbo": 2.3848549891740847, "distortion": 1.303570615939605, "elbo": 2.3848549891740847, "rate": 1.0812843579512377} +{"time": "Thu Nov 12 04:36:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 134, "step": 1340000, "distortion": 1.3021836454868316, "distortion_nans": 0.0, "elbo": 2.3859777657985686, "elbo_filtered": 2.3859777657985686, "rate": 1.0837941228151322, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.31762211227417, "grad_norm": 23.131427764892578} +{"time": "Thu Nov 12 04:36:12 2020", "message": "Saving model@ 1340000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 04:36:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1340000, "distortion": 1.3021630722284316, "distortion_nans": 0.0, "elbo": 2.3859285316467287, "elbo_filtered": 2.3859285316467287, "rate": 1.0837654619216919, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176670348644257, "grad_norm": 23.131427764892578} +{"time": "Thu Nov 12 04:36:28 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1340000.png"} +{"time": "Thu Nov 12 04:58:24 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1341000, "distortion": 1.2992267330884935, "distortion_nans": 0.0, "elbo": 2.3848267867565154, "elbo_filtered": 2.3848267867565154, "rate": 1.0856000541448594, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3142210955619813, "grad_norm": 20.11134910583496} +{"time": "Thu Nov 12 05:20:15 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1342000, "distortion": 1.2973162565231324, "distortion_nans": 0.0, "elbo": 2.38733553647995, "elbo_filtered": 2.38733553647995, "rate": 1.090019287109375, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3099500210285187, "grad_norm": 18.801095962524414} +{"time": "Thu Nov 12 05:42:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1343000, "distortion": 1.2984419540166854, "distortion_nans": 0.0, "elbo": 2.3887128393650054, "elbo_filtered": 2.3887128393650054, "rate": 1.090270888209343, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3152715256214143, "grad_norm": 18.066699981689453} +{"time": "Thu Nov 12 06:04:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1344000, "distortion": 1.300353743314743, "distortion_nans": 0.0, "elbo": 2.386442085981369, "elbo_filtered": 2.386442085981369, "rate": 1.0860883414149285, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.326722909927368, "grad_norm": 22.412687301635742} +{"time": "Thu Nov 12 06:26:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1345000, "distortion": 1.2978754894733429, "distortion_nans": 0.0, "elbo": 2.382743362426758, "elbo_filtered": 2.382743362426758, "rate": 1.0848678737878799, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3171945867538453, "grad_norm": 19.395383834838867} +{"time": "Thu Nov 12 06:48:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1346000, "distortion": 1.2983251987695694, "distortion_nans": 0.0, "elbo": 2.388182935476303, "elbo_filtered": 2.388182935476303, "rate": 1.0898577356338501, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3196959483623505, "grad_norm": 20.441883087158203} +{"time": "Thu Nov 12 07:10:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1347000, "distortion": 1.2993391598463058, "distortion_nans": 0.0, "elbo": 2.3859833459854127, "elbo_filtered": 2.3859833459854127, "rate": 1.0866441838741303, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3205774748325347, "grad_norm": 23.935056686401367} +{"time": "Thu Nov 12 07:32:08 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1348000, "distortion": 1.2969131007194519, "distortion_nans": 0.0, "elbo": 2.384398942232132, "elbo_filtered": 2.384398942232132, "rate": 1.0874858556985856, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3085073494911195, "grad_norm": 21.113525390625} +{"time": "Thu Nov 12 07:54:07 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 134, "step": 1349000, "distortion": 1.2961282559633256, "distortion_nans": 0.0, "elbo": 2.386096055030823, "elbo_filtered": 2.386096055030823, "rate": 1.0899678000211717, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3178605043888092, "grad_norm": 20.07445526123047} +{"time": "Thu Nov 12 08:14:10 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 134, "step": 1349910, "n_batches": 39, "filtered_elbo": 2.383882926060603, "distortion": 1.2972407982899592, "elbo": 2.383882926060603, "rate": 1.086642088034214} +{"time": "Thu Nov 12 08:16:07 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 135, "step": 1350000, "distortion": 1.2969782421588898, "distortion_nans": 0.0, "elbo": 2.386238311767578, "elbo_filtered": 2.386238311767578, "rate": 1.0892600684165954, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.31238023853302, "grad_norm": 22.304500579833984} +{"time": "Thu Nov 12 08:16:07 2020", "message": "Saving model@ 1350000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 08:16:26 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1350000, "distortion": 1.2969401372671128, "distortion_nans": 0.0, "elbo": 2.3862118079662324, "elbo_filtered": 2.3862118079662324, "rate": 1.089271669626236, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3123748762607574, "grad_norm": 22.304500579833984} +{"time": "Thu Nov 12 08:16:28 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1350000.png"} +{"time": "Thu Nov 12 08:38:35 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1351000, "distortion": 1.2947382729053498, "distortion_nans": 0.0, "elbo": 2.3848968954086303, "elbo_filtered": 2.3848968954086303, "rate": 1.0901586260199547, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.325691871881485, "grad_norm": 18.579954147338867} +{"time": "Thu Nov 12 09:00:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1352000, "distortion": 1.2916527035236358, "distortion_nans": 0.0, "elbo": 2.3849505026340485, "elbo_filtered": 2.3849505026340485, "rate": 1.093297809958458, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3227093002796173, "grad_norm": 19.499452590942383} +{"time": "Thu Nov 12 09:22:30 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1353000, "distortion": 1.2956008157730103, "distortion_nans": 0.0, "elbo": 2.386725817203522, "elbo_filtered": 2.386725817203522, "rate": 1.0911250054836272, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3109938175678253, "grad_norm": 20.997793197631836} +{"time": "Thu Nov 12 09:44:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1354000, "distortion": 1.294309061050415, "distortion_nans": 0.0, "elbo": 2.384837065935135, "elbo_filtered": 2.384837065935135, "rate": 1.0905280009508134, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.322380960226059, "grad_norm": 23.816617965698242} +{"time": "Thu Nov 12 10:06:32 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1355000, "distortion": 1.293076144695282, "distortion_nans": 0.0, "elbo": 2.3855773422718047, "elbo_filtered": 2.3855773422718047, "rate": 1.092501193165779, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3179040522575378, "grad_norm": 22.654441833496094} +{"time": "Thu Nov 12 10:28:37 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1356000, "distortion": 1.294864344239235, "distortion_nans": 0.0, "elbo": 2.384558908224106, "elbo_filtered": 2.384558908224106, "rate": 1.0896945598125458, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.324074868440628, "grad_norm": 19.44154930114746} +{"time": "Thu Nov 12 10:50:31 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1357000, "distortion": 1.2920656700134276, "distortion_nans": 0.0, "elbo": 2.385634073495865, "elbo_filtered": 2.385634073495865, "rate": 1.0935684106349945, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3127554941177368, "grad_norm": 19.0850830078125} +{"time": "Thu Nov 12 11:12:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1358000, "distortion": 1.2914035520553588, "distortion_nans": 0.0, "elbo": 2.385010917901993, "elbo_filtered": 2.385010917901993, "rate": 1.0936073591709137, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.315260615825653, "grad_norm": 20.82229232788086} +{"time": "Thu Nov 12 11:34:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 135, "step": 1359000, "distortion": 1.2964507822990416, "distortion_nans": 0.0, "elbo": 2.3867002117633818, "elbo_filtered": 2.3867002117633818, "rate": 1.090249426960945, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3201406252384187, "grad_norm": 20.998184204101562} +{"time": "Thu Nov 12 11:53:46 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 135, "step": 1359880, "n_batches": 39, "filtered_elbo": 2.3843371929266515, "distortion": 1.293670614560445, "elbo": 2.3843371929266515, "rate": 1.0906665783662062} +{"time": "Thu Nov 12 11:56:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 136, "step": 1360000, "distortion": 1.291126739025116, "distortion_nans": 0.0, "elbo": 2.3857937417030333, "elbo_filtered": 2.3857937417030333, "rate": 1.0946670039892197, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.309473204612732, "grad_norm": 18.193578720092773} +{"time": "Thu Nov 12 11:56:25 2020", "message": "Saving model@ 1360000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 11:56:40 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1360000, "distortion": 1.2911498064994813, "distortion_nans": 0.0, "elbo": 2.3858332018852235, "elbo_filtered": 2.3858332018852235, "rate": 1.094683396935463, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3095534250736236, "grad_norm": 18.193578720092773} +{"time": "Thu Nov 12 11:56:41 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1360000.png"} +{"time": "Thu Nov 12 12:18:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1361000, "distortion": 1.2920363914966584, "distortion_nans": 0.0, "elbo": 2.385304571866989, "elbo_filtered": 2.385304571866989, "rate": 1.0932681875228882, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3112842626571655, "grad_norm": 19.877662658691406} +{"time": "Thu Nov 12 12:40:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1362000, "distortion": 1.290185727596283, "distortion_nans": 0.0, "elbo": 2.3831762104034424, "elbo_filtered": 2.3831762104034424, "rate": 1.092990490436554, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3235978956222534, "grad_norm": 22.058612823486328} +{"time": "Thu Nov 12 13:02:36 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1363000, "distortion": 1.290865690946579, "distortion_nans": 0.0, "elbo": 2.384993847608566, "elbo_filtered": 2.384993847608566, "rate": 1.0941281534433365, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3169951605796815, "grad_norm": 19.9332275390625} +{"time": "Thu Nov 12 13:24:47 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1364000, "distortion": 1.292685725569725, "distortion_nans": 0.0, "elbo": 2.387208212375641, "elbo_filtered": 2.387208212375641, "rate": 1.0945224857330322, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3301890318393708, "grad_norm": 18.865020751953125} +{"time": "Thu Nov 12 13:46:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1365000, "distortion": 1.2950412479639053, "distortion_nans": 0.0, "elbo": 2.3889943935871125, "elbo_filtered": 2.3889943935871125, "rate": 1.0939531474113464, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3221129434108734, "grad_norm": 21.59498405456543} +{"time": "Thu Nov 12 14:08:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1366000, "distortion": 1.2921341978311538, "distortion_nans": 0.0, "elbo": 2.3850010464191436, "elbo_filtered": 2.3850010464191436, "rate": 1.092866854429245, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3133567123413086, "grad_norm": 72.33169555664062} +{"time": "Thu Nov 12 14:30:26 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1367000, "distortion": 1.2919683185815811, "distortion_nans": 0.0, "elbo": 2.3857221586704256, "elbo_filtered": 2.3857221586704256, "rate": 1.0937538439035415, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.300670238494873, "grad_norm": 21.01091766357422} +{"time": "Thu Nov 12 14:52:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1368000, "distortion": 1.2900984669923783, "distortion_nans": 0.0, "elbo": 2.38333225440979, "elbo_filtered": 2.38333225440979, "rate": 1.0932337857484817, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3101811308860778, "grad_norm": 19.155677795410156} +{"time": "Thu Nov 12 15:14:05 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 136, "step": 1369000, "distortion": 1.2923250374794006, "distortion_nans": 0.0, "elbo": 2.3860721151828765, "elbo_filtered": 2.3860721151828765, "rate": 1.0937470737695694, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3066296510696411, "grad_norm": 21.70737648010254} +{"time": "Thu Nov 12 15:32:49 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 136, "step": 1369850, "n_batches": 39, "filtered_elbo": 2.384452129021669, "distortion": 1.292567286735926, "elbo": 2.384452129021669, "rate": 1.0918848025493133} +{"time": "Thu Nov 12 15:36:01 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 137, "step": 1370000, "distortion": 1.2924724740982056, "distortion_nans": 0.0, "elbo": 2.3862389080524444, "elbo_filtered": 2.3862389080524444, "rate": 1.0937664277553558, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3090890178680419, "grad_norm": 21.965425491333008} +{"time": "Thu Nov 12 15:36:01 2020", "message": "Saving model@ 1370000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 15:36:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1370000, "distortion": 1.2924727569818497, "distortion_nans": 0.0, "elbo": 2.386186900854111, "elbo_filtered": 2.386186900854111, "rate": 1.0937141375541688, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3092082183361053, "grad_norm": 21.965425491333008} +{"time": "Thu Nov 12 15:36:17 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1370000.png"} +{"time": "Thu Nov 12 15:58:22 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1371000, "distortion": 1.2937199662923813, "distortion_nans": 0.0, "elbo": 2.3856176528930666, "elbo_filtered": 2.3856176528930666, "rate": 1.0918976916074752, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323285598039627, "grad_norm": 46.18595504760742} +{"time": "Thu Nov 12 16:20:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1372000, "distortion": 1.2906406239271164, "distortion_nans": 0.0, "elbo": 2.384469544649124, "elbo_filtered": 2.384469544649124, "rate": 1.0938289231061935, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3140905623435974, "grad_norm": 17.803848266601562} +{"time": "Thu Nov 12 16:42:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1373000, "distortion": 1.2890050090551377, "distortion_nans": 0.0, "elbo": 2.383037755489349, "elbo_filtered": 2.383037755489349, "rate": 1.0940327433347703, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3117333793640136, "grad_norm": 19.153486251831055} +{"time": "Thu Nov 12 17:04:02 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1374000, "distortion": 1.2919322142601013, "distortion_nans": 0.0, "elbo": 2.3892891194820405, "elbo_filtered": 2.3892891194820405, "rate": 1.0973569084405899, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.311059420824051, "grad_norm": 20.89454460144043} +{"time": "Thu Nov 12 17:26:05 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1375000, "distortion": 1.28800035572052, "distortion_nans": 0.0, "elbo": 2.383549860715866, "elbo_filtered": 2.383549860715866, "rate": 1.0955495110750197, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3180365607738496, "grad_norm": 21.401403427124023} +{"time": "Thu Nov 12 17:48:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1376000, "distortion": 1.2907392050027848, "distortion_nans": 0.0, "elbo": 2.384592829704285, "elbo_filtered": 2.384592829704285, "rate": 1.0938536195755004, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3167713708877564, "grad_norm": 32.44023132324219} +{"time": "Thu Nov 12 18:10:00 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1377000, "distortion": 1.2915268512964249, "distortion_nans": 0.0, "elbo": 2.3883211896419527, "elbo_filtered": 2.3883211896419527, "rate": 1.0967943375110627, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3162592689990997, "grad_norm": 22.331348419189453} +{"time": "Thu Nov 12 18:32:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1378000, "distortion": 1.289876671075821, "distortion_nans": 0.0, "elbo": 2.3850198283195496, "elbo_filtered": 2.3850198283195496, "rate": 1.0951431589126588, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3218555819988251, "grad_norm": 21.830745697021484} +{"time": "Thu Nov 12 18:54:02 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 137, "step": 1379000, "distortion": 1.290222186088562, "distortion_nans": 0.0, "elbo": 2.38648738694191, "elbo_filtered": 2.38648738694191, "rate": 1.0962651976346969, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3177147414684296, "grad_norm": 22.95673370361328} +{"time": "Thu Nov 12 19:12:09 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 137, "step": 1379820, "n_batches": 39, "filtered_elbo": 2.384204399891389, "distortion": 1.2894886120771751, "elbo": 2.384204399891389, "rate": 1.0947158581171281} +{"time": "Thu Nov 12 19:16:07 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 138, "step": 1380000, "distortion": 1.2848004660606385, "distortion_nans": 0.0, "elbo": 2.3823525938987733, "elbo_filtered": 2.3823525938987733, "rate": 1.0975521210432053, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317813364982605, "grad_norm": 19.435150146484375} +{"time": "Thu Nov 12 19:16:07 2020", "message": "Saving model@ 1380000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 19:16:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1380000, "distortion": 1.2847897449731827, "distortion_nans": 0.0, "elbo": 2.382376301765442, "elbo_filtered": 2.382376301765442, "rate": 1.0975865498781203, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317992844581604, "grad_norm": 19.435150146484375} +{"time": "Thu Nov 12 19:16:23 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1380000.png"} +{"time": "Thu Nov 12 19:38:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1381000, "distortion": 1.284948206782341, "distortion_nans": 0.0, "elbo": 2.383818853855133, "elbo_filtered": 2.383818853855133, "rate": 1.0988706446886063, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3228100819587707, "grad_norm": 21.478679656982422} +{"time": "Thu Nov 12 20:00:26 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1382000, "distortion": 1.2848066643476486, "distortion_nans": 0.0, "elbo": 2.3823590166568755, "elbo_filtered": 2.3823590166568755, "rate": 1.0975523475408555, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176877760887147, "grad_norm": 19.15582275390625} +{"time": "Thu Nov 12 20:22:20 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1383000, "distortion": 1.2857105295658111, "distortion_nans": 0.0, "elbo": 2.384239095211029, "elbo_filtered": 2.384239095211029, "rate": 1.0985285665988922, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.313770369529724, "grad_norm": 26.155261993408203} +{"time": "Thu Nov 12 20:44:24 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1384000, "distortion": 1.2863084565401077, "distortion_nans": 0.0, "elbo": 2.383988070964813, "elbo_filtered": 2.383988070964813, "rate": 1.0976796211004258, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3226610481739045, "grad_norm": 20.50432586669922} +{"time": "Thu Nov 12 21:06:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1385000, "distortion": 1.287855794787407, "distortion_nans": 0.0, "elbo": 2.3853055534362793, "elbo_filtered": 2.3853055534362793, "rate": 1.0974497628211974, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3233165636062623, "grad_norm": 19.34364891052246} +{"time": "Thu Nov 12 21:28:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1386000, "distortion": 1.2878780608177185, "distortion_nans": 0.0, "elbo": 2.384284348964691, "elbo_filtered": 2.384284348964691, "rate": 1.0964062905311585, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3202946290969848, "grad_norm": 22.704299926757812} +{"time": "Thu Nov 12 21:50:35 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1387000, "distortion": 1.2883402928113938, "distortion_nans": 0.0, "elbo": 2.385682789325714, "elbo_filtered": 2.385682789325714, "rate": 1.0973424974679946, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3244555685520172, "grad_norm": 24.131032943725586} +{"time": "Thu Nov 12 22:12:32 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1388000, "distortion": 1.2898308138847352, "distortion_nans": 0.0, "elbo": 2.3873158099651337, "elbo_filtered": 2.3873158099651337, "rate": 1.0974849939346314, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3160505285263062, "grad_norm": 18.533321380615234} +{"time": "Thu Nov 12 22:34:26 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 138, "step": 1389000, "distortion": 1.2893324928283691, "distortion_nans": 0.0, "elbo": 2.3863340022563935, "elbo_filtered": 2.3863340022563935, "rate": 1.0970015044212342, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3133050785064697, "grad_norm": 25.11432647705078} +{"time": "Thu Nov 12 22:52:00 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 138, "step": 1389790, "n_batches": 39, "filtered_elbo": 2.383526209073189, "distortion": 1.2900698551764855, "elbo": 2.383526209073189, "rate": 1.0934563263868675} +{"time": "Thu Nov 12 22:56:37 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 139, "step": 1390000, "distortion": 1.2901702626943587, "distortion_nans": 0.0, "elbo": 2.388439674139023, "elbo_filtered": 2.388439674139023, "rate": 1.0982694073915482, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3236626808643341, "grad_norm": 39.05070114135742} +{"time": "Thu Nov 12 22:56:37 2020", "message": "Saving model@ 1390000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Thu Nov 12 22:56:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1390000, "distortion": 1.2901434129476548, "distortion_nans": 0.0, "elbo": 2.388413237333298, "elbo_filtered": 2.388413237333298, "rate": 1.0982698200941086, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323574415922165, "grad_norm": 39.05070114135742} +{"time": "Thu Nov 12 22:56:53 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1390000.png"} +{"time": "Thu Nov 12 23:18:47 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1391000, "distortion": 1.285820076227188, "distortion_nans": 0.0, "elbo": 2.3849871044158935, "elbo_filtered": 2.3849871044158935, "rate": 1.0991670351028442, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3133653790950774, "grad_norm": 18.024011611938477} +{"time": "Thu Nov 12 23:40:54 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1392000, "distortion": 1.286866826891899, "distortion_nans": 0.0, "elbo": 2.38413219499588, "elbo_filtered": 2.38413219499588, "rate": 1.0972653725147248, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.326016984462738, "grad_norm": 89.8430404663086} +{"time": "Fri Nov 13 00:02:49 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1393000, "distortion": 1.2898266912698746, "distortion_nans": 0.0, "elbo": 2.38725008225441, "elbo_filtered": 2.38725008225441, "rate": 1.0974234005212784, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3132072796821594, "grad_norm": 50.90227508544922} +{"time": "Fri Nov 13 00:24:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1394000, "distortion": 1.2870498093366622, "distortion_nans": 0.0, "elbo": 2.385515481233597, "elbo_filtered": 2.385515481233597, "rate": 1.0984656771421433, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3269287574291229, "grad_norm": 33.62545394897461} +{"time": "Fri Nov 13 00:46:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1395000, "distortion": 1.2875707317590714, "distortion_nans": 0.0, "elbo": 2.3850266730785368, "elbo_filtered": 2.3850266730785368, "rate": 1.0974559433460236, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3156227996349334, "grad_norm": 22.969688415527344} +{"time": "Fri Nov 13 01:08:57 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1396000, "distortion": 1.2897378064393996, "distortion_nans": 0.0, "elbo": 2.386942174911499, "elbo_filtered": 2.386942174911499, "rate": 1.0972043660879136, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3229719305038452, "grad_norm": 24.10704803466797} +{"time": "Fri Nov 13 01:30:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1397000, "distortion": 1.2864151066541671, "distortion_nans": 0.0, "elbo": 2.3821405079364775, "elbo_filtered": 2.3821405079364775, "rate": 1.095725395321846, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.31381676530838, "grad_norm": 20.999717712402344} +{"time": "Fri Nov 13 01:52:54 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1398000, "distortion": 1.289479578256607, "distortion_nans": 0.0, "elbo": 2.3846666429042815, "elbo_filtered": 2.3846666429042815, "rate": 1.0951870630979539, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3214339292049408, "grad_norm": 122.44791412353516} +{"time": "Fri Nov 13 02:14:54 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 139, "step": 1399000, "distortion": 1.2841700389385224, "distortion_nans": 0.0, "elbo": 2.3829576802253722, "elbo_filtered": 2.3829576802253722, "rate": 1.0987876406908035, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3186755833625794, "grad_norm": 20.380006790161133} +{"time": "Fri Nov 13 02:31:42 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 139, "step": 1399760, "n_batches": 39, "filtered_elbo": 2.3841989773970385, "distortion": 1.2877346399502876, "elbo": 2.3841989773970385, "rate": 1.0964642946536725} +{"time": "Fri Nov 13 02:37:00 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 140, "step": 1400000, "distortion": 1.2847362065315246, "distortion_nans": 0.0, "elbo": 2.3853522124290465, "elbo_filtered": 2.3853522124290465, "rate": 1.1006160136461258, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3188663239479066, "grad_norm": 27.263208389282227} +{"time": "Fri Nov 13 02:37:00 2020", "message": "Saving model@ 1400000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 02:37:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1400000, "distortion": 1.2846934312582017, "distortion_nans": 0.0, "elbo": 2.3852824358940126, "elbo_filtered": 2.3852824358940126, "rate": 1.1005890122652053, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318836204767227, "grad_norm": 27.263208389282227} +{"time": "Fri Nov 13 02:37:20 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1400000.png"} +{"time": "Fri Nov 13 02:59:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1401000, "distortion": 1.2869071210622787, "distortion_nans": 0.0, "elbo": 2.386075707912445, "elbo_filtered": 2.386075707912445, "rate": 1.099168595433235, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3198407809734345, "grad_norm": 23.013254165649414} +{"time": "Fri Nov 13 03:21:32 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1402000, "distortion": 1.2856866886615754, "distortion_nans": 0.0, "elbo": 2.385415753364563, "elbo_filtered": 2.385415753364563, "rate": 1.0997290688753127, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3295619127750398, "grad_norm": 26.794010162353516} +{"time": "Fri Nov 13 03:43:31 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1403000, "distortion": 1.2863130527734756, "distortion_nans": 0.0, "elbo": 2.383612823724747, "elbo_filtered": 2.383612823724747, "rate": 1.0972997704744338, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3187716302871704, "grad_norm": 55.026920318603516} +{"time": "Fri Nov 13 04:05:30 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1404000, "distortion": 1.2847275562286378, "distortion_nans": 0.0, "elbo": 2.3819063944816588, "elbo_filtered": 2.3819063944816588, "rate": 1.097178843021393, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176458976268768, "grad_norm": 20.478561401367188} +{"time": "Fri Nov 13 04:27:36 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1405000, "distortion": 1.285443234205246, "distortion_nans": 0.0, "elbo": 2.387397340774536, "elbo_filtered": 2.387397340774536, "rate": 1.1019541021585464, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3246748712062835, "grad_norm": 19.370500564575195} +{"time": "Fri Nov 13 04:49:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1406000, "distortion": 1.2854608497619628, "distortion_nans": 0.0, "elbo": 2.3860807173252105, "elbo_filtered": 2.3860807173252105, "rate": 1.1006198716163635, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3272344541549683, "grad_norm": 18.974748611450195} +{"time": "Fri Nov 13 05:11:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1407000, "distortion": 1.2848885387182236, "distortion_nans": 0.0, "elbo": 2.3853919115066526, "elbo_filtered": 2.3853919115066526, "rate": 1.1005033737421035, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3248243596553801, "grad_norm": 22.396564483642578} +{"time": "Fri Nov 13 05:33:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1408000, "distortion": 1.2846891629695891, "distortion_nans": 0.0, "elbo": 2.385745835542679, "elbo_filtered": 2.385745835542679, "rate": 1.1010566771030426, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.313842715024948, "grad_norm": 23.658679962158203} +{"time": "Fri Nov 13 05:55:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 140, "step": 1409000, "distortion": 1.282961207985878, "distortion_nans": 0.0, "elbo": 2.3835367119312285, "elbo_filtered": 2.3835367119312285, "rate": 1.1005755107402801, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.324607311487198, "grad_norm": 101.25997161865234} +{"time": "Fri Nov 13 06:11:50 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 140, "step": 1409730, "n_batches": 39, "filtered_elbo": 2.384092526558118, "distortion": 1.285725046426822, "elbo": 2.384092526558118, "rate": 1.0983675137544289} +{"time": "Fri Nov 13 06:17:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 141, "step": 1410000, "distortion": 1.283015549659729, "distortion_nans": 0.0, "elbo": 2.383041965961456, "elbo_filtered": 2.383041965961456, "rate": 1.1000264155864716, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3147687175273894, "grad_norm": 29.982545852661133} +{"time": "Fri Nov 13 06:17:52 2020", "message": "Saving model@ 1410000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 06:18:06 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1410000, "distortion": 1.2831050678491593, "distortion_nans": 0.0, "elbo": 2.3831833753585814, "elbo_filtered": 2.3831833753585814, "rate": 1.1000783067941666, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3147298080921173, "grad_norm": 29.982545852661133} +{"time": "Fri Nov 13 06:18:08 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1410000.png"} +{"time": "Fri Nov 13 06:40:13 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1411000, "distortion": 1.2848776866197587, "distortion_nans": 0.0, "elbo": 2.386854442834854, "elbo_filtered": 2.386854442834854, "rate": 1.1019767607450486, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.324052895784378, "grad_norm": 262.3968200683594} +{"time": "Fri Nov 13 07:02:30 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1412000, "distortion": 1.2789451384544372, "distortion_nans": 0.0, "elbo": 2.38151092672348, "elbo_filtered": 2.38151092672348, "rate": 1.102565784931183, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.335706790447235, "grad_norm": 18.21790885925293} +{"time": "Fri Nov 13 07:24:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1413000, "distortion": 1.2832884792089463, "distortion_nans": 0.0, "elbo": 2.3854859240055086, "elbo_filtered": 2.3854859240055086, "rate": 1.1021974453926087, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3104695279598235, "grad_norm": 24.92432403564453} +{"time": "Fri Nov 13 07:46:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1414000, "distortion": 1.284316033244133, "distortion_nans": 0.0, "elbo": 2.382411840438843, "elbo_filtered": 2.382411840438843, "rate": 1.0980958136320114, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3189785425662994, "grad_norm": 28.132808685302734} +{"time": "Fri Nov 13 08:08:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1415000, "distortion": 1.2856748234033584, "distortion_nans": 0.0, "elbo": 2.384118929862976, "elbo_filtered": 2.384118929862976, "rate": 1.0984441088438035, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3232178621292114, "grad_norm": 19.968374252319336} +{"time": "Fri Nov 13 08:30:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1416000, "distortion": 1.28462688601017, "distortion_nans": 0.0, "elbo": 2.3841771790981294, "elbo_filtered": 2.3841771790981294, "rate": 1.0995502895116807, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3354123587608338, "grad_norm": 20.98041534423828} +{"time": "Fri Nov 13 08:52:40 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1417000, "distortion": 1.2860651593208312, "distortion_nans": 0.0, "elbo": 2.3852029881477357, "elbo_filtered": 2.3852029881477357, "rate": 1.0991378306150437, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317656324148178, "grad_norm": 34.406707763671875} +{"time": "Fri Nov 13 09:14:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1418000, "distortion": 1.2870568515062333, "distortion_nans": 0.0, "elbo": 2.3863873364925383, "elbo_filtered": 2.3863873364925383, "rate": 1.0993304804563522, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3249338099956511, "grad_norm": 21.17928695678711} +{"time": "Fri Nov 13 09:36:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 141, "step": 1419000, "distortion": 1.288569304585457, "distortion_nans": 0.0, "elbo": 2.387920873641968, "elbo_filtered": 2.387920873641968, "rate": 1.0993515684604644, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318985624551773, "grad_norm": 63.6641731262207} +{"time": "Fri Nov 13 09:52:21 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 141, "step": 1419700, "n_batches": 39, "filtered_elbo": 2.384500509653336, "distortion": 1.2874069336133125, "elbo": 2.384500509653336, "rate": 1.0970935699267266} +{"time": "Fri Nov 13 09:58:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 142, "step": 1420000, "distortion": 1.2847066801786422, "distortion_nans": 0.0, "elbo": 2.3847016773223877, "elbo_filtered": 2.3847016773223877, "rate": 1.0999950034618378, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3233327865600586, "grad_norm": 25.110305786132812} +{"time": "Fri Nov 13 09:58:56 2020", "message": "Saving model@ 1420000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 09:59:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1420000, "distortion": 1.284748034119606, "distortion_nans": 0.0, "elbo": 2.3847376036643984, "elbo_filtered": 2.3847376036643984, "rate": 1.099989576101303, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3233922271728515, "grad_norm": 25.110305786132812} +{"time": "Fri Nov 13 09:59:12 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1420000.png"} +{"time": "Fri Nov 13 10:21:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1421000, "distortion": 1.2866464669704438, "distortion_nans": 0.0, "elbo": 2.3874475140571594, "elbo_filtered": 2.3874475140571594, "rate": 1.1008010510206223, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318419016122818, "grad_norm": 24.489076614379883} +{"time": "Fri Nov 13 10:43:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1422000, "distortion": 1.281326761841774, "distortion_nans": 0.0, "elbo": 2.3839909467697145, "elbo_filtered": 2.3839909467697145, "rate": 1.102664182305336, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3186082170009612, "grad_norm": 18.274105072021484} +{"time": "Fri Nov 13 11:05:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1423000, "distortion": 1.2836149941682815, "distortion_nans": 0.0, "elbo": 2.384321854352951, "elbo_filtered": 2.384321854352951, "rate": 1.100706858754158, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3257053594589234, "grad_norm": 24.386436462402344} +{"time": "Fri Nov 13 11:27:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1424000, "distortion": 1.278836947798729, "distortion_nans": 0.0, "elbo": 2.3797850375175478, "elbo_filtered": 2.3797850375175478, "rate": 1.1009480931758882, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3191666297912599, "grad_norm": 47.406253814697266} +{"time": "Fri Nov 13 11:49:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1425000, "distortion": 1.2806265614032746, "distortion_nans": 0.0, "elbo": 2.384027366876602, "elbo_filtered": 2.384027366876602, "rate": 1.1034007999897004, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.313047078371048, "grad_norm": 54.83747482299805} +{"time": "Fri Nov 13 12:11:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1426000, "distortion": 1.2833144071102143, "distortion_nans": 0.0, "elbo": 2.385556731462479, "elbo_filtered": 2.385556731462479, "rate": 1.1022423236370087, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3185314118862153, "grad_norm": 21.938302993774414} +{"time": "Fri Nov 13 12:33:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1427000, "distortion": 1.282326626777649, "distortion_nans": 0.0, "elbo": 2.3845353965759277, "elbo_filtered": 2.3845353965759277, "rate": 1.1022087730169297, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3209074852466582, "grad_norm": 19.31660270690918} +{"time": "Fri Nov 13 12:55:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1428000, "distortion": 1.283861634016037, "distortion_nans": 0.0, "elbo": 2.387272602081299, "elbo_filtered": 2.387272602081299, "rate": 1.1034109694957732, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3214514775276185, "grad_norm": 20.059541702270508} +{"time": "Fri Nov 13 13:17:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 142, "step": 1429000, "distortion": 1.2826483271121978, "distortion_nans": 0.0, "elbo": 2.3830680651664733, "elbo_filtered": 2.3830680651664733, "rate": 1.100419734954834, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.310050145626068, "grad_norm": 137.38070678710938} +{"time": "Fri Nov 13 13:32:04 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 142, "step": 1429670, "n_batches": 39, "filtered_elbo": 2.3828381330539017, "distortion": 1.286932578453651, "elbo": 2.3828381330539017, "rate": 1.0959055851667354} +{"time": "Fri Nov 13 13:39:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 143, "step": 1430000, "distortion": 1.2879863479137421, "distortion_nans": 0.0, "elbo": 2.3871440198421476, "elbo_filtered": 2.3871440198421476, "rate": 1.0991576795578002, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3257313973903655, "grad_norm": 43.255645751953125} +{"time": "Fri Nov 13 13:39:25 2020", "message": "Saving model@ 1430000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 13:39:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1430000, "distortion": 1.2880080523490907, "distortion_nans": 0.0, "elbo": 2.387169068098068, "elbo_filtered": 2.387169068098068, "rate": 1.0991610236167908, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3258628044128418, "grad_norm": 43.255645751953125} +{"time": "Fri Nov 13 13:39:41 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1430000.png"} +{"time": "Fri Nov 13 14:01:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1431000, "distortion": 1.2852815469503402, "distortion_nans": 0.0, "elbo": 2.384751033306122, "elbo_filtered": 2.384751033306122, "rate": 1.0994694917201995, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.322904221534729, "grad_norm": 104.22526550292969} +{"time": "Fri Nov 13 14:23:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1432000, "distortion": 1.2811923077106475, "distortion_nans": 0.0, "elbo": 2.3841347460746767, "elbo_filtered": 2.3841347460746767, "rate": 1.1029424403905868, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3194821763038636, "grad_norm": 23.11030387878418} +{"time": "Fri Nov 13 14:45:47 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1433000, "distortion": 1.2802264111042023, "distortion_nans": 0.0, "elbo": 2.3832116963863372, "elbo_filtered": 2.3832116963863372, "rate": 1.1029852806329727, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3209251525402068, "grad_norm": 37.3387336730957} +{"time": "Fri Nov 13 15:07:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1434000, "distortion": 1.2857568842172622, "distortion_nans": 0.0, "elbo": 2.3859175820350647, "elbo_filtered": 2.3859175820350647, "rate": 1.10016070497036, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3234939968585968, "grad_norm": 125.15087890625} +{"time": "Fri Nov 13 15:29:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1435000, "distortion": 1.2841730827093125, "distortion_nans": 0.0, "elbo": 2.3838303053379057, "elbo_filtered": 2.3838303053379057, "rate": 1.0996572222709655, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3190879647731781, "grad_norm": 30.002843856811523} +{"time": "Fri Nov 13 15:51:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1436000, "distortion": 1.286317193031311, "distortion_nans": 0.0, "elbo": 2.3857037422657013, "elbo_filtered": 2.3857037422657013, "rate": 1.0993865514993668, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3172465751171112, "grad_norm": 23.511409759521484} +{"time": "Fri Nov 13 16:13:55 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1437000, "distortion": 1.285325479388237, "distortion_nans": 0.0, "elbo": 2.3864466304779053, "elbo_filtered": 2.3864466304779053, "rate": 1.1011211540699004, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3241230528354644, "grad_norm": 19.657691955566406} +{"time": "Fri Nov 13 16:35:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1438000, "distortion": 1.2854833264350891, "distortion_nans": 0.0, "elbo": 2.3845016934871675, "elbo_filtered": 2.3845016934871675, "rate": 1.0990183628797532, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3161400022506713, "grad_norm": 39.93986129760742} +{"time": "Fri Nov 13 16:57:58 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 143, "step": 1439000, "distortion": 1.2858599840402603, "distortion_nans": 0.0, "elbo": 2.387061607837677, "elbo_filtered": 2.387061607837677, "rate": 1.101201629281044, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3250083980560303, "grad_norm": 122.08290100097656} +{"time": "Fri Nov 13 17:12:07 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 143, "step": 1439640, "n_batches": 39, "filtered_elbo": 2.3828398019839554, "distortion": 1.2852333172773704, "elbo": 2.3828398019839554, "rate": 1.0976064877632337} +{"time": "Fri Nov 13 17:20:02 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 144, "step": 1440000, "distortion": 1.282045679807663, "distortion_nans": 0.0, "elbo": 2.382980481863022, "elbo_filtered": 2.382980481863022, "rate": 1.1009348106384278, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317113354921341, "grad_norm": 38.7335319519043} +{"time": "Fri Nov 13 17:20:02 2020", "message": "Saving model@ 1440000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 17:20:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1440000, "distortion": 1.2821682798862457, "distortion_nans": 0.0, "elbo": 2.383180006980896, "elbo_filtered": 2.383180006980896, "rate": 1.1010117356777191, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3169381000995637, "grad_norm": 38.7335319519043} +{"time": "Fri Nov 13 17:20:18 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1440000.png"} +{"time": "Fri Nov 13 17:42:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1441000, "distortion": 1.2872718349695205, "distortion_nans": 0.0, "elbo": 2.382872478723526, "elbo_filtered": 2.382872478723526, "rate": 1.0956006404161454, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3218146138191222, "grad_norm": 110.05662536621094} +{"time": "Fri Nov 13 18:04:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1442000, "distortion": 1.2838794264793396, "distortion_nans": 0.0, "elbo": 2.385671142578125, "elbo_filtered": 2.385671142578125, "rate": 1.101791723370552, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.326175788640976, "grad_norm": 18.901103973388672} +{"time": "Fri Nov 13 18:26:33 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1443000, "distortion": 1.2821325722932815, "distortion_nans": 0.0, "elbo": 2.3843945796489714, "elbo_filtered": 2.3843945796489714, "rate": 1.1022620112895967, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3238771331310273, "grad_norm": 28.226436614990234} +{"time": "Fri Nov 13 18:48:37 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1444000, "distortion": 1.2807474880218506, "distortion_nans": 0.0, "elbo": 2.3823326699733736, "elbo_filtered": 2.3823326699733736, "rate": 1.1015851808786392, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3223380990028382, "grad_norm": 53.748477935791016} +{"time": "Fri Nov 13 19:10:48 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1445000, "distortion": 1.2827593126296997, "distortion_nans": 0.0, "elbo": 2.384222515821457, "elbo_filtered": 2.384222515821457, "rate": 1.1014632091522216, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3300431125164032, "grad_norm": 67.39372253417969} +{"time": "Fri Nov 13 19:32:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1446000, "distortion": 1.2807800555229187, "distortion_nans": 0.0, "elbo": 2.3833525242805482, "elbo_filtered": 2.3833525242805482, "rate": 1.1025724730491637, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3241428523063659, "grad_norm": 23.753482818603516} +{"time": "Fri Nov 13 19:54:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1447000, "distortion": 1.285107243180275, "distortion_nans": 0.0, "elbo": 2.387849924325943, "elbo_filtered": 2.387849924325943, "rate": 1.10274268078804, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3191431922912598, "grad_norm": 24.4400691986084} +{"time": "Fri Nov 13 20:16:57 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1448000, "distortion": 1.2814986243247986, "distortion_nans": 0.0, "elbo": 2.382998419046402, "elbo_filtered": 2.382998419046402, "rate": 1.1014997990131379, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3237961597442627, "grad_norm": 55.820594787597656} +{"time": "Fri Nov 13 20:38:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 144, "step": 1449000, "distortion": 1.2818167873620987, "distortion_nans": 0.0, "elbo": 2.3871994473934173, "elbo_filtered": 2.3871994473934173, "rate": 1.1053826624155045, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317383006811142, "grad_norm": 63.48561096191406} +{"time": "Fri Nov 13 20:52:20 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 144, "step": 1449610, "n_batches": 39, "filtered_elbo": 2.3834210481399145, "distortion": 1.2843304505715003, "elbo": 2.3834210481399145, "rate": 1.0990905914551172} +{"time": "Fri Nov 13 21:01:00 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 145, "step": 1450000, "distortion": 1.2841087640523912, "distortion_nans": 0.0, "elbo": 2.385417308807373, "elbo_filtered": 2.385417308807373, "rate": 1.1013085452318192, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3166488761901856, "grad_norm": 18.780160903930664} +{"time": "Fri Nov 13 21:01:00 2020", "message": "Saving model@ 1450000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Fri Nov 13 21:01:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1450000, "distortion": 1.2841074339151382, "distortion_nans": 0.0, "elbo": 2.3854035811424255, "elbo_filtered": 2.3854035811424255, "rate": 1.1012961480617522, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3167218112945556, "grad_norm": 18.780160903930664} +{"time": "Fri Nov 13 21:01:21 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1450000.png"} +{"time": "Fri Nov 13 21:23:20 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1451000, "distortion": 1.2804611321687698, "distortion_nans": 0.0, "elbo": 2.3837990539073943, "elbo_filtered": 2.3837990539073943, "rate": 1.1033379219770432, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3186540071964263, "grad_norm": 25.295934677124023} +{"time": "Fri Nov 13 21:45:24 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1452000, "distortion": 1.2809049741029739, "distortion_nans": 0.0, "elbo": 2.383055382013321, "elbo_filtered": 2.383055382013321, "rate": 1.1021504102945328, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3227431857585907, "grad_norm": 20.90761947631836} +{"time": "Fri Nov 13 22:07:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1453000, "distortion": 1.2840849677324295, "distortion_nans": 0.0, "elbo": 2.383878724575043, "elbo_filtered": 2.383878724575043, "rate": 1.0997937597036362, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3199204483032227, "grad_norm": 34.625362396240234} +{"time": "Fri Nov 13 22:29:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1454000, "distortion": 1.2817402341365813, "distortion_nans": 0.0, "elbo": 2.3838654260635375, "elbo_filtered": 2.3838654260635375, "rate": 1.102125198841095, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3122228207588196, "grad_norm": 20.397666931152344} +{"time": "Fri Nov 13 22:51:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1455000, "distortion": 1.279911430478096, "distortion_nans": 0.0, "elbo": 2.3844222223758695, "elbo_filtered": 2.3844222223758695, "rate": 1.104510787010193, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3178760569095611, "grad_norm": 76.43414306640625} +{"time": "Fri Nov 13 23:13:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1456000, "distortion": 1.2839725090265275, "distortion_nans": 0.0, "elbo": 2.385407283306122, "elbo_filtered": 2.385407283306122, "rate": 1.101434779047966, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3231777398586273, "grad_norm": 213.57342529296875} +{"time": "Fri Nov 13 23:35:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1457000, "distortion": 1.2819722130298614, "distortion_nans": 0.0, "elbo": 2.3841601984500884, "elbo_filtered": 2.3841601984500884, "rate": 1.1021879856586456, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3146516079902648, "grad_norm": 50.0328369140625} +{"time": "Fri Nov 13 23:57:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1458000, "distortion": 1.2788292902708054, "distortion_nans": 0.0, "elbo": 2.3832033216953277, "elbo_filtered": 2.3832033216953277, "rate": 1.1043740305900573, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3149480459690095, "grad_norm": 19.602094650268555} +{"time": "Sat Nov 14 00:19:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 145, "step": 1459000, "distortion": 1.2854055399894715, "distortion_nans": 0.0, "elbo": 2.387494896888733, "elbo_filtered": 2.387494896888733, "rate": 1.10208935713768, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.322245088338852, "grad_norm": 23.860517501831055} +{"time": "Sat Nov 14 00:32:01 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 145, "step": 1459580, "n_batches": 39, "filtered_elbo": 2.383262200233264, "distortion": 1.2827988251661644, "elbo": 2.383262200233264, "rate": 1.100463411746881} +{"time": "Sat Nov 14 00:41:14 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 146, "step": 1460000, "distortion": 1.2813998723030091, "distortion_nans": 0.0, "elbo": 2.3851201288700103, "elbo_filtered": 2.3851201288700103, "rate": 1.103720253109932, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3108481738567352, "grad_norm": 19.861141204833984} +{"time": "Sat Nov 14 00:41:14 2020", "message": "Saving model@ 1460000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 00:41:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1460000, "distortion": 1.2813342305421829, "distortion_nans": 0.0, "elbo": 2.3850389220714567, "elbo_filtered": 2.3850389220714567, "rate": 1.1037046880722046, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3108335025310516, "grad_norm": 19.861141204833984} +{"time": "Sat Nov 14 00:41:30 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1460000.png"} +{"time": "Sat Nov 14 01:03:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1461000, "distortion": 1.2783254393339156, "distortion_nans": 0.0, "elbo": 2.3821568577289582, "elbo_filtered": 2.3821568577289582, "rate": 1.103831417798996, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3045397458076478, "grad_norm": 22.386363983154297} +{"time": "Sat Nov 14 01:25:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1462000, "distortion": 1.2799772617816925, "distortion_nans": 0.0, "elbo": 2.383533487319946, "elbo_filtered": 2.383533487319946, "rate": 1.1035562261343002, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.321524108171463, "grad_norm": 116.28504943847656} +{"time": "Sat Nov 14 01:47:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1463000, "distortion": 1.282295215845108, "distortion_nans": 0.0, "elbo": 2.3847225852012635, "elbo_filtered": 2.3847225852012635, "rate": 1.1024273681640624, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3262039680480957, "grad_norm": 301.1042785644531} +{"time": "Sat Nov 14 02:09:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1464000, "distortion": 1.279553726911545, "distortion_nans": 0.0, "elbo": 2.381853075504303, "elbo_filtered": 2.381853075504303, "rate": 1.1022993459701538, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3191245436668395, "grad_norm": 25.49905776977539} +{"time": "Sat Nov 14 02:31:32 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1465000, "distortion": 1.2787401125431062, "distortion_nans": 0.0, "elbo": 2.384075996875763, "elbo_filtered": 2.384075996875763, "rate": 1.1053358857631683, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.32584210562706, "grad_norm": 24.55851173400879} +{"time": "Sat Nov 14 02:53:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1466000, "distortion": 1.2807031064033507, "distortion_nans": 0.0, "elbo": 2.385739617586136, "elbo_filtered": 2.385739617586136, "rate": 1.1050365059375762, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3044393725395202, "grad_norm": 30.723356246948242} +{"time": "Sat Nov 14 03:15:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1467000, "distortion": 1.283207330584526, "distortion_nans": 0.0, "elbo": 2.3863851554393767, "elbo_filtered": 2.3863851554393767, "rate": 1.1031778297424317, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318431819677353, "grad_norm": 203.72715759277344} +{"time": "Sat Nov 14 03:37:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1468000, "distortion": 1.282863879084587, "distortion_nans": 0.0, "elbo": 2.3854535474777223, "elbo_filtered": 2.3854535474777223, "rate": 1.102589670419693, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3198800053596496, "grad_norm": 42.422080993652344} +{"time": "Sat Nov 14 03:59:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 146, "step": 1469000, "distortion": 1.2824452347755433, "distortion_nans": 0.0, "elbo": 2.385759932518005, "elbo_filtered": 2.385759932518005, "rate": 1.10331469810009, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176000492572784, "grad_norm": 213.21531677246094} +{"time": "Sat Nov 14 04:11:27 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 146, "step": 1469550, "n_batches": 39, "filtered_elbo": 2.383546003928551, "distortion": 1.2826330081010475, "elbo": 2.383546003928551, "rate": 1.1009129988841522} +{"time": "Sat Nov 14 04:21:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 147, "step": 1470000, "distortion": 1.2800134023427963, "distortion_nans": 0.0, "elbo": 2.3824499480724333, "elbo_filtered": 2.3824499480724333, "rate": 1.1024365561008453, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3243171064853667, "grad_norm": 25.792577743530273} +{"time": "Sat Nov 14 04:21:27 2020", "message": "Saving model@ 1470000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 04:21:42 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1470000, "distortion": 1.2800941121578218, "distortion_nans": 0.0, "elbo": 2.382527865409851, "elbo_filtered": 2.382527865409851, "rate": 1.1024337635040282, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3242210237979888, "grad_norm": 25.792577743530273} +{"time": "Sat Nov 14 04:21:44 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1470000.png"} +{"time": "Sat Nov 14 04:43:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1471000, "distortion": 1.2776875281333924, "distortion_nans": 0.0, "elbo": 2.3828933682441713, "elbo_filtered": 2.3828933682441713, "rate": 1.1052058379650116, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3130726504325867, "grad_norm": 54.62086486816406} +{"time": "Sat Nov 14 05:05:37 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1472000, "distortion": 1.280511664867401, "distortion_nans": 0.0, "elbo": 2.38533785367012, "elbo_filtered": 2.38533785367012, "rate": 1.1048261857032775, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176810882091523, "grad_norm": 140.27159118652344} +{"time": "Sat Nov 14 05:27:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1473000, "distortion": 1.2761052976846694, "distortion_nans": 0.0, "elbo": 2.380558968782425, "elbo_filtered": 2.380558968782425, "rate": 1.1044536699056626, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3160596721172333, "grad_norm": 125.48441314697266} +{"time": "Sat Nov 14 05:49:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1474000, "distortion": 1.27682770216465, "distortion_nans": 0.0, "elbo": 2.385155397415161, "elbo_filtered": 2.385155397415161, "rate": 1.1083277000188827, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.314155061006546, "grad_norm": 20.496051788330078} +{"time": "Sat Nov 14 06:11:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1475000, "distortion": 1.2811028261184692, "distortion_nans": 0.0, "elbo": 2.385619305610657, "elbo_filtered": 2.385619305610657, "rate": 1.1045164748430252, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323674090385437, "grad_norm": 76.33454132080078} +{"time": "Sat Nov 14 06:33:48 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1476000, "distortion": 1.278337450146675, "distortion_nans": 0.0, "elbo": 2.384617091655731, "elbo_filtered": 2.384617091655731, "rate": 1.1062796449661254, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.329103944540024, "grad_norm": 21.7468318939209} +{"time": "Sat Nov 14 06:55:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1477000, "distortion": 1.2796463807821274, "distortion_nans": 0.0, "elbo": 2.3847024137973785, "elbo_filtered": 2.3847024137973785, "rate": 1.105056041121483, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3224255023002625, "grad_norm": 50.35335922241211} +{"time": "Sat Nov 14 07:17:47 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1478000, "distortion": 1.2778946108818054, "distortion_nans": 0.0, "elbo": 2.3847824420928956, "elbo_filtered": 2.3847824420928956, "rate": 1.1068878382444383, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3148457100391389, "grad_norm": 23.934995651245117} +{"time": "Sat Nov 14 07:39:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 147, "step": 1479000, "distortion": 1.2806483566761018, "distortion_nans": 0.0, "elbo": 2.3846918642520905, "elbo_filtered": 2.3846918642520905, "rate": 1.1040435034036635, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3106446661949158, "grad_norm": 24.67416763305664} +{"time": "Sat Nov 14 07:51:12 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 147, "step": 1479520, "n_batches": 39, "filtered_elbo": 2.3828525971143675, "distortion": 1.2821231744228265, "elbo": 2.3828525971143675, "rate": 1.1007294288048377} +{"time": "Sat Nov 14 08:01:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 148, "step": 1480000, "distortion": 1.282117733001709, "distortion_nans": 0.0, "elbo": 2.3861401772499082, "elbo_filtered": 2.3861401772499082, "rate": 1.1040224434137345, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318471578836441, "grad_norm": 131.70730590820312} +{"time": "Sat Nov 14 08:01:44 2020", "message": "Saving model@ 1480000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 08:01:59 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1480000, "distortion": 1.2820777326822281, "distortion_nans": 0.0, "elbo": 2.386077881574631, "elbo_filtered": 2.386077881574631, "rate": 1.1040001479387282, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3185017004013062, "grad_norm": 131.70730590820312} +{"time": "Sat Nov 14 08:02:01 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1480000.png"} +{"time": "Sat Nov 14 08:24:01 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1481000, "distortion": 1.278763200521469, "distortion_nans": 0.0, "elbo": 2.382178382396698, "elbo_filtered": 2.382178382396698, "rate": 1.1034151871800422, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318996421813965, "grad_norm": 27.277070999145508} +{"time": "Sat Nov 14 08:45:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1482000, "distortion": 1.2788237799406053, "distortion_nans": 0.0, "elbo": 2.384225924730301, "elbo_filtered": 2.384225924730301, "rate": 1.1054021450281144, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3147581527233123, "grad_norm": 61.80298614501953} +{"time": "Sat Nov 14 09:07:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1483000, "distortion": 1.2812493642568588, "distortion_nans": 0.0, "elbo": 2.3826538062095644, "elbo_filtered": 2.3826538062095644, "rate": 1.1014044412374497, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3081277277469634, "grad_norm": 317.8031005859375} +{"time": "Sat Nov 14 09:29:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1484000, "distortion": 1.280380437731743, "distortion_nans": 0.0, "elbo": 2.3846754591464996, "elbo_filtered": 2.3846754591464996, "rate": 1.1042950254678727, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3238110609054565, "grad_norm": 23.513790130615234} +{"time": "Sat Nov 14 09:51:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1485000, "distortion": 1.2796536680459976, "distortion_nans": 0.0, "elbo": 2.3840176599025726, "elbo_filtered": 2.3840176599025726, "rate": 1.104364000082016, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3150097341537477, "grad_norm": 23.297243118286133} +{"time": "Sat Nov 14 10:13:43 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1486000, "distortion": 1.2802747611999512, "distortion_nans": 0.0, "elbo": 2.384487629890442, "elbo_filtered": 2.384487629890442, "rate": 1.104212864279747, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3159530699253081, "grad_norm": 68.75633239746094} +{"time": "Sat Nov 14 10:35:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1487000, "distortion": 1.2787311531305312, "distortion_nans": 0.0, "elbo": 2.383548907518387, "elbo_filtered": 2.383548907518387, "rate": 1.1048177537918091, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3207533438205719, "grad_norm": 20.934612274169922} +{"time": "Sat Nov 14 10:57:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1488000, "distortion": 1.27937457549572, "distortion_nans": 0.0, "elbo": 2.384986775636673, "elbo_filtered": 2.384986775636673, "rate": 1.1056122070550918, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3133817884922028, "grad_norm": 40.852027893066406} +{"time": "Sat Nov 14 11:19:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 148, "step": 1489000, "distortion": 1.281623234987259, "distortion_nans": 0.0, "elbo": 2.385876478910446, "elbo_filtered": 2.385876478910446, "rate": 1.1042532485723495, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3181235897541046, "grad_norm": 43.511959075927734} +{"time": "Sat Nov 14 11:30:32 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 148, "step": 1489490, "n_batches": 39, "filtered_elbo": 2.3833034955538235, "distortion": 1.2820042218917456, "elbo": 2.3833034955538235, "rate": 1.1012992950586171} +{"time": "Sat Nov 14 11:41:48 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 149, "step": 1490000, "distortion": 1.2811925382614135, "distortion_nans": 0.0, "elbo": 2.3855633027553558, "elbo_filtered": 2.3855633027553558, "rate": 1.104370766401291, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3219995694160462, "grad_norm": 22.591217041015625} +{"time": "Sat Nov 14 11:41:48 2020", "message": "Saving model@ 1490000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 11:42:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1490000, "distortion": 1.2811506375074386, "distortion_nans": 0.0, "elbo": 2.385474818468094, "elbo_filtered": 2.385474818468094, "rate": 1.104324182868004, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3222317550182343, "grad_norm": 22.591217041015625} +{"time": "Sat Nov 14 11:42:04 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1490000.png"} +{"time": "Sat Nov 14 12:04:08 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1491000, "distortion": 1.2779120790958405, "distortion_nans": 0.0, "elbo": 2.3833660953044893, "elbo_filtered": 2.3833660953044893, "rate": 1.1054540227651597, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3227798523902894, "grad_norm": 28.006309509277344} +{"time": "Sat Nov 14 12:26:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1492000, "distortion": 1.2779467734098435, "distortion_nans": 0.0, "elbo": 2.381630662202835, "elbo_filtered": 2.381630662202835, "rate": 1.1036838954687118, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.328196333885193, "grad_norm": 36.580867767333984} +{"time": "Sat Nov 14 12:48:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1493000, "distortion": 1.2784601911306381, "distortion_nans": 0.0, "elbo": 2.3820989422798156, "elbo_filtered": 2.3820989422798156, "rate": 1.1036387547254563, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3309248993396758, "grad_norm": 19.31561279296875} +{"time": "Sat Nov 14 13:10:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1494000, "distortion": 1.2787092299461364, "distortion_nans": 0.0, "elbo": 2.3841314582824706, "elbo_filtered": 2.3841314582824706, "rate": 1.1054222314357758, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3173547585010528, "grad_norm": 82.11651611328125} +{"time": "Sat Nov 14 13:32:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1495000, "distortion": 1.2811501749753953, "distortion_nans": 0.0, "elbo": 2.385499661922455, "elbo_filtered": 2.385499661922455, "rate": 1.1043494848012925, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.332502480983734, "grad_norm": 75.37394714355469} +{"time": "Sat Nov 14 13:54:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1496000, "distortion": 1.2798573228120804, "distortion_nans": 0.0, "elbo": 2.386272423028946, "elbo_filtered": 2.386272423028946, "rate": 1.1064151083230973, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3176824967861176, "grad_norm": 21.844314575195312} +{"time": "Sat Nov 14 14:16:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1497000, "distortion": 1.2801751059293747, "distortion_nans": 0.0, "elbo": 2.387738777399063, "elbo_filtered": 2.387738777399063, "rate": 1.1075636769533157, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318433518409729, "grad_norm": 23.425657272338867} +{"time": "Sat Nov 14 14:38:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1498000, "distortion": 1.2803724936246872, "distortion_nans": 0.0, "elbo": 2.3838712849617005, "elbo_filtered": 2.3838712849617005, "rate": 1.1034987921714783, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3214016530513764, "grad_norm": 27.713430404663086} +{"time": "Sat Nov 14 15:00:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 149, "step": 1499000, "distortion": 1.2786062306165695, "distortion_nans": 0.0, "elbo": 2.38260289478302, "elbo_filtered": 2.38260289478302, "rate": 1.1039966636896132, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.319477639913559, "grad_norm": 42.67737579345703} +{"time": "Sat Nov 14 15:10:56 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 149, "step": 1499460, "n_batches": 39, "filtered_elbo": 2.3827643333337245, "distortion": 1.2804575791725745, "elbo": 2.3827643333337245, "rate": 1.1023067969542284} +{"time": "Sat Nov 14 15:22:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 150, "step": 1500000, "distortion": 1.276202821493149, "distortion_nans": 0.0, "elbo": 2.383604153394699, "elbo_filtered": 2.383604153394699, "rate": 1.1074013413190842, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3223008008003234, "grad_norm": 21.015987396240234} +{"time": "Sat Nov 14 15:22:51 2020", "message": "Saving model@ 1500000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 15:23:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1500000, "distortion": 1.276244383454323, "distortion_nans": 0.0, "elbo": 2.3836860978603363, "elbo_filtered": 2.3836860978603363, "rate": 1.1074417238235474, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3223635206222535, "grad_norm": 21.015987396240234} +{"time": "Sat Nov 14 15:23:12 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1500000.png"} +{"time": "Sat Nov 14 15:45:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1501000, "distortion": 1.2775972505807878, "distortion_nans": 0.0, "elbo": 2.386390561580658, "elbo_filtered": 2.386390561580658, "rate": 1.1087933146953584, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.32478471159935, "grad_norm": 22.863487243652344} +{"time": "Sat Nov 14 16:07:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1502000, "distortion": 1.2776347448825836, "distortion_nans": 0.0, "elbo": 2.3798028330802916, "elbo_filtered": 2.3798028330802916, "rate": 1.1021680878400804, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.304520493030548, "grad_norm": 22.0405330657959} +{"time": "Sat Nov 14 16:28:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1503000, "distortion": 1.2797052488327025, "distortion_nans": 0.0, "elbo": 2.3866982235908507, "elbo_filtered": 2.3866982235908507, "rate": 1.106992974281311, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3116195282936096, "grad_norm": 54.84505081176758} +{"time": "Sat Nov 14 16:50:42 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1504000, "distortion": 1.2764089723825456, "distortion_nans": 0.0, "elbo": 2.3826822369098664, "elbo_filtered": 2.3826822369098664, "rate": 1.106273263335228, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.305161376953125, "grad_norm": 33.29227066040039} +{"time": "Sat Nov 14 17:12:37 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1505000, "distortion": 1.277517725110054, "distortion_nans": 0.0, "elbo": 2.382499719142914, "elbo_filtered": 2.382499719142914, "rate": 1.10498200237751, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3143370966911316, "grad_norm": 22.445018768310547} +{"time": "Sat Nov 14 17:34:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1506000, "distortion": 1.2773866451978684, "distortion_nans": 0.0, "elbo": 2.384452260017395, "elbo_filtered": 2.384452260017395, "rate": 1.1070656183958054, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.2974920527935028, "grad_norm": 25.910078048706055} +{"time": "Sat Nov 14 17:56:09 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1507000, "distortion": 1.2795958495140076, "distortion_nans": 0.0, "elbo": 2.3834594881534574, "elbo_filtered": 2.3834594881534574, "rate": 1.1038636413812637, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3121611044406891, "grad_norm": 288.0616455078125} +{"time": "Sat Nov 14 18:17:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1508000, "distortion": 1.2791202702522277, "distortion_nans": 0.0, "elbo": 2.383948524236679, "elbo_filtered": 2.383948524236679, "rate": 1.1048282593488694, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3029921419620514, "grad_norm": 59.484371185302734} +{"time": "Sat Nov 14 18:39:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 150, "step": 1509000, "distortion": 1.2797749521732331, "distortion_nans": 0.0, "elbo": 2.384335418701172, "elbo_filtered": 2.384335418701172, "rate": 1.1045604630708694, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3121840138435363, "grad_norm": 51.82758331298828} +{"time": "Sat Nov 14 18:49:13 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 150, "step": 1509430, "n_batches": 39, "filtered_elbo": 2.382835027499077, "distortion": 1.280828622671274, "elbo": 2.382835027499077, "rate": 1.1020063895445604} +{"time": "Sat Nov 14 19:01:36 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 151, "step": 1510000, "distortion": 1.2789741086959838, "distortion_nans": 0.0, "elbo": 2.3843344180583954, "elbo_filtered": 2.3843344180583954, "rate": 1.1053603050708771, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3029959766864776, "grad_norm": 93.88262176513672} +{"time": "Sat Nov 14 19:01:36 2020", "message": "Saving model@ 1510000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 19:01:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1510000, "distortion": 1.2789119027853013, "distortion_nans": 0.0, "elbo": 2.3842379758358003, "elbo_filtered": 2.3842379758358003, "rate": 1.1053260688781739, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3031712172031402, "grad_norm": 93.88262176513672} +{"time": "Sat Nov 14 19:01:52 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1510000.png"} +{"time": "Sat Nov 14 19:23:43 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1511000, "distortion": 1.2819772448539735, "distortion_nans": 0.0, "elbo": 2.385924669742584, "elbo_filtered": 2.385924669742584, "rate": 1.1039474300146104, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3091000549793244, "grad_norm": 49.23752975463867} +{"time": "Sat Nov 14 19:45:31 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1512000, "distortion": 1.2805907731056214, "distortion_nans": 0.0, "elbo": 2.3885799596309663, "elbo_filtered": 2.3885799596309663, "rate": 1.1079891854524613, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3071154868602752, "grad_norm": 25.374238967895508} +{"time": "Sat Nov 14 20:07:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1513000, "distortion": 1.282095424413681, "distortion_nans": 0.0, "elbo": 2.3837937145233155, "elbo_filtered": 2.3837937145233155, "rate": 1.1016982867717744, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.315052540063858, "grad_norm": 260.9143981933594} +{"time": "Sat Nov 14 20:29:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1514000, "distortion": 1.2846243492364884, "distortion_nans": 0.0, "elbo": 2.3869021983146665, "elbo_filtered": 2.3869021983146665, "rate": 1.1022778483629228, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3210477604866029, "grad_norm": 109.92047119140625} +{"time": "Sat Nov 14 20:51:28 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1515000, "distortion": 1.2786277997493745, "distortion_nans": 0.0, "elbo": 2.381862993001938, "elbo_filtered": 2.381862993001938, "rate": 1.1032352011203765, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3182055869102478, "grad_norm": 65.9501724243164} +{"time": "Sat Nov 14 21:13:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1516000, "distortion": 1.2784452959299089, "distortion_nans": 0.0, "elbo": 2.3824098954200745, "elbo_filtered": 2.3824098954200745, "rate": 1.1039645975828172, "rate_nans": 0.0, "skipped_updates": 1, "iter_time": 1.3187704601287842, "grad_norm": 418.2236328125} +{"time": "Sat Nov 14 21:35:21 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1517000, "distortion": 1.277164398789406, "distortion_nans": 0.0, "elbo": 2.3824289140701294, "elbo_filtered": 2.3824289140701294, "rate": 1.1052645143270492, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.31240279006958, "grad_norm": 20.761188507080078} +{"time": "Sat Nov 14 21:57:09 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1518000, "distortion": 1.2756824399232864, "distortion_nans": 0.0, "elbo": 2.3820426619052886, "elbo_filtered": 2.3820426619052886, "rate": 1.106360219478607, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3074836483001708, "grad_norm": 60.67499542236328} +{"time": "Sat Nov 14 22:18:55 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 151, "step": 1519000, "distortion": 1.2735121909379958, "distortion_nans": 0.0, "elbo": 2.380714838027954, "elbo_filtered": 2.380714838027954, "rate": 1.1072026525735854, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3045560569763184, "grad_norm": 21.546836853027344} +{"time": "Sat Nov 14 22:27:48 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 151, "step": 1519400, "n_batches": 39, "filtered_elbo": 2.3822865852942834, "distortion": 1.2780281152480688, "elbo": 2.3822865852942834, "rate": 1.1042584639329176} +{"time": "Sat Nov 14 22:41:02 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 152, "step": 1520000, "distortion": 1.2792268185615538, "distortion_nans": 0.0, "elbo": 2.385370099067688, "elbo_filtered": 2.385370099067688, "rate": 1.1061432787179948, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3204619340896606, "grad_norm": 36.13795852661133} +{"time": "Sat Nov 14 22:41:02 2020", "message": "Saving model@ 1520000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sat Nov 14 22:41:17 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1520000, "distortion": 1.2792600396871567, "distortion_nans": 0.0, "elbo": 2.3854050278663634, "elbo_filtered": 2.3854050278663634, "rate": 1.1061449862718582, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3205701196193695, "grad_norm": 36.13795852661133} +{"time": "Sat Nov 14 22:41:19 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1520000.png"} +{"time": "Sat Nov 14 23:03:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1521000, "distortion": 1.2768277740478515, "distortion_nans": 0.0, "elbo": 2.3824566254615784, "elbo_filtered": 2.3824566254615784, "rate": 1.1056288548707962, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3115363614559175, "grad_norm": 115.2656021118164} +{"time": "Sat Nov 14 23:25:08 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1522000, "distortion": 1.2760096118450164, "distortion_nans": 0.0, "elbo": 2.383988627433777, "elbo_filtered": 2.383988627433777, "rate": 1.1079790110588075, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3151073935031892, "grad_norm": 28.360698699951172} +{"time": "Sat Nov 14 23:47:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1523000, "distortion": 1.2752761195898057, "distortion_nans": 0.0, "elbo": 2.3820957169532777, "elbo_filtered": 2.3820957169532777, "rate": 1.1068195995092391, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3210966908931732, "grad_norm": 326.6656494140625} +{"time": "Sun Nov 15 00:09:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1524000, "distortion": 1.2764724171161652, "distortion_nans": 0.0, "elbo": 2.3821307594776155, "elbo_filtered": 2.3821307594776155, "rate": 1.1056583372354507, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3129183394908905, "grad_norm": 22.250669479370117} +{"time": "Sun Nov 15 00:31:02 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1525000, "distortion": 1.2794477480649948, "distortion_nans": 0.0, "elbo": 2.3848637840747835, "elbo_filtered": 2.3848637840747835, "rate": 1.105416046500206, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3131381945610046, "grad_norm": 20.475643157958984} +{"time": "Sun Nov 15 00:52:54 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1526000, "distortion": 1.275172918200493, "distortion_nans": 0.0, "elbo": 2.3826464879512788, "elbo_filtered": 2.3826464879512788, "rate": 1.107473571419716, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3104810264110565, "grad_norm": 21.811540603637695} +{"time": "Sun Nov 15 01:14:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1527000, "distortion": 1.2770761506557464, "distortion_nans": 0.0, "elbo": 2.386019900560379, "elbo_filtered": 2.386019900560379, "rate": 1.1089437536001205, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3189772243499756, "grad_norm": 27.1468448638916} +{"time": "Sun Nov 15 01:36:58 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1528000, "distortion": 1.2807398021221161, "distortion_nans": 0.0, "elbo": 2.3872193319797517, "elbo_filtered": 2.3872193319797517, "rate": 1.1064795383214951, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323314641714096, "grad_norm": 24.677608489990234} +{"time": "Sun Nov 15 01:58:53 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 152, "step": 1529000, "distortion": 1.2769506640434265, "distortion_nans": 0.0, "elbo": 2.3819345500469207, "elbo_filtered": 2.3819345500469207, "rate": 1.1049838871955873, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.314630557537079, "grad_norm": 106.44234466552734} +{"time": "Sun Nov 15 02:07:13 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 152, "step": 1529370, "n_batches": 39, "filtered_elbo": 2.382263495371892, "distortion": 1.2784149738458486, "elbo": 2.382263495371892, "rate": 1.1038484970728557} +{"time": "Sun Nov 15 02:21:01 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 153, "step": 1530000, "distortion": 1.2772213612794876, "distortion_nans": 0.0, "elbo": 2.386538418292999, "elbo_filtered": 2.386538418292999, "rate": 1.1093170592784882, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.320447916507721, "grad_norm": 21.36622428894043} +{"time": "Sun Nov 15 02:21:01 2020", "message": "Saving model@ 1530000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 02:21:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1530000, "distortion": 1.277184450507164, "distortion_nans": 0.0, "elbo": 2.386485307216644, "elbo_filtered": 2.386485307216644, "rate": 1.1093008588552475, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3203177318572998, "grad_norm": 21.36622428894043} +{"time": "Sun Nov 15 02:21:17 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1530000.png"} +{"time": "Sun Nov 15 02:43:12 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1531000, "distortion": 1.2747632179260253, "distortion_nans": 0.0, "elbo": 2.383983354330063, "elbo_filtered": 2.383983354330063, "rate": 1.109220139503479, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3132774233818054, "grad_norm": 41.14521789550781} +{"time": "Sun Nov 15 03:05:08 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1532000, "distortion": 1.2769160732030869, "distortion_nans": 0.0, "elbo": 2.386041111707687, "elbo_filtered": 2.386041111707687, "rate": 1.1091250368356704, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.315652850151062, "grad_norm": 19.976179122924805} +{"time": "Sun Nov 15 03:27:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1533000, "distortion": 1.2790474208593368, "distortion_nans": 0.0, "elbo": 2.3846197469234465, "elbo_filtered": 2.3846197469234465, "rate": 1.1055723278522491, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3210562562942505, "grad_norm": 28.729076385498047} +{"time": "Sun Nov 15 03:49:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1534000, "distortion": 1.2756109902858734, "distortion_nans": 0.0, "elbo": 2.382952576637268, "elbo_filtered": 2.382952576637268, "rate": 1.107341583967209, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3246814596652985, "grad_norm": 24.775035858154297} +{"time": "Sun Nov 15 04:11:18 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1535000, "distortion": 1.2778202409744264, "distortion_nans": 0.0, "elbo": 2.3833228261470794, "elbo_filtered": 2.3833228261470794, "rate": 1.105502588391304, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3210704553127288, "grad_norm": 20.129148483276367} +{"time": "Sun Nov 15 04:33:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1536000, "distortion": 1.2758108651638032, "distortion_nans": 0.0, "elbo": 2.380755393266678, "elbo_filtered": 2.380755393266678, "rate": 1.1049445320367812, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3199896659851074, "grad_norm": 21.54945945739746} +{"time": "Sun Nov 15 04:55:19 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1537000, "distortion": 1.2766853339672088, "distortion_nans": 0.0, "elbo": 2.3831817665100097, "elbo_filtered": 2.3831817665100097, "rate": 1.1064964319467545, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.319558577299118, "grad_norm": 215.49407958984375} +{"time": "Sun Nov 15 05:17:23 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1538000, "distortion": 1.2727659024000169, "distortion_nans": 0.0, "elbo": 2.3821923582553866, "elbo_filtered": 2.3821923582553866, "rate": 1.1094264552593232, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3234733641147614, "grad_norm": 44.908199310302734} +{"time": "Sun Nov 15 05:39:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 153, "step": 1539000, "distortion": 1.2740214624404906, "distortion_nans": 0.0, "elbo": 2.383656008720398, "elbo_filtered": 2.383656008720398, "rate": 1.1096345435380937, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3225134143829345, "grad_norm": 106.07003784179688} +{"time": "Sun Nov 15 05:47:03 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 153, "step": 1539340, "n_batches": 39, "filtered_elbo": 2.3816146789452968, "distortion": 1.2744973042072394, "elbo": 2.3816146789452968, "rate": 1.107117393077948} +{"time": "Sun Nov 15 06:01:35 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 154, "step": 1540000, "distortion": 1.273639691233635, "distortion_nans": 0.0, "elbo": 2.3831002230644227, "elbo_filtered": 2.3831002230644227, "rate": 1.1094605333805083, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3213484537601472, "grad_norm": 21.35994529724121} +{"time": "Sun Nov 15 06:01:35 2020", "message": "Saving model@ 1540000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 06:01:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1540000, "distortion": 1.2735885150432587, "distortion_nans": 0.0, "elbo": 2.3830186808109284, "elbo_filtered": 2.3830186808109284, "rate": 1.1094301669597626, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3212621212005615, "grad_norm": 21.35994529724121} +{"time": "Sun Nov 15 06:01:52 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1540000.png"} +{"time": "Sun Nov 15 06:24:04 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1541000, "distortion": 1.2766312690973283, "distortion_nans": 0.0, "elbo": 2.383502348661423, "elbo_filtered": 2.383502348661423, "rate": 1.1068710852861405, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3314412760734557, "grad_norm": 65.0875244140625} +{"time": "Sun Nov 15 06:46:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1542000, "distortion": 1.2731153521537781, "distortion_nans": 0.0, "elbo": 2.3798029692173004, "elbo_filtered": 2.3798029692173004, "rate": 1.1066876208782197, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3263053562641143, "grad_norm": 120.1964340209961} +{"time": "Sun Nov 15 07:08:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1543000, "distortion": 1.273752902984619, "distortion_nans": 0.0, "elbo": 2.3805350606441498, "elbo_filtered": 2.3805350606441498, "rate": 1.1067821505069733, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3187027456760407, "grad_norm": 21.78363037109375} +{"time": "Sun Nov 15 07:30:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1544000, "distortion": 1.273734903693199, "distortion_nans": 0.0, "elbo": 2.3825104384422304, "elbo_filtered": 2.3825104384422304, "rate": 1.108775536298752, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3195188097953796, "grad_norm": 20.84320068359375} +{"time": "Sun Nov 15 07:52:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1545000, "distortion": 1.2783071839809417, "distortion_nans": 0.0, "elbo": 2.3865381853580474, "elbo_filtered": 2.3865381853580474, "rate": 1.1082310050725936, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3232641656398774, "grad_norm": 194.42984008789062} +{"time": "Sun Nov 15 08:14:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1546000, "distortion": 1.2769443769454956, "distortion_nans": 0.0, "elbo": 2.3852866187095643, "elbo_filtered": 2.3852866187095643, "rate": 1.1083422446250915, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3147278506755828, "grad_norm": 24.943082809448242} +{"time": "Sun Nov 15 08:36:11 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1547000, "distortion": 1.2819068273305894, "distortion_nans": 0.0, "elbo": 2.386765176296234, "elbo_filtered": 2.386765176296234, "rate": 1.1048583600521087, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.319211436033249, "grad_norm": 369.11419677734375} +{"time": "Sun Nov 15 08:58:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1548000, "distortion": 1.2761339771747588, "distortion_nans": 0.0, "elbo": 2.3834067168235777, "elbo_filtered": 2.3834067168235777, "rate": 1.107272744178772, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3234745099544525, "grad_norm": 25.483062744140625} +{"time": "Sun Nov 15 09:20:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 154, "step": 1549000, "distortion": 1.2736071482896805, "distortion_nans": 0.0, "elbo": 2.3813095269203184, "elbo_filtered": 2.3813095269203184, "rate": 1.1077023735046387, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3129384636878967, "grad_norm": 41.10854721069336} +{"time": "Sun Nov 15 09:27:05 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 154, "step": 1549310, "n_batches": 39, "filtered_elbo": 2.382031110616831, "distortion": 1.278534540763268, "elbo": 2.382031110616831, "rate": 1.1034965820801563} +{"time": "Sun Nov 15 09:42:14 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 155, "step": 1550000, "distortion": 1.2766090774536132, "distortion_nans": 0.0, "elbo": 2.383100152730942, "elbo_filtered": 2.383100152730942, "rate": 1.1064910762310027, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3171435377597809, "grad_norm": 166.78402709960938} +{"time": "Sun Nov 15 09:42:14 2020", "message": "Saving model@ 1550000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 09:42:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1550000, "distortion": 1.2766311259269714, "distortion_nans": 0.0, "elbo": 2.383077573299408, "elbo_filtered": 2.383077573299408, "rate": 1.106446448326111, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.317233857154846, "grad_norm": 166.78402709960938} +{"time": "Sun Nov 15 09:42:35 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1550000.png"} +{"time": "Sun Nov 15 10:04:43 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1551000, "distortion": 1.277686550140381, "distortion_nans": 0.0, "elbo": 2.3857071034908293, "elbo_filtered": 2.3857071034908293, "rate": 1.1080205528736113, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3267518408298493, "grad_norm": 24.806610107421875} +{"time": "Sun Nov 15 10:26:41 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1552000, "distortion": 1.27955246758461, "distortion_nans": 0.0, "elbo": 2.3850723991394043, "elbo_filtered": 2.3850723991394043, "rate": 1.1055199295282363, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3172129361629485, "grad_norm": 65.82184600830078} +{"time": "Sun Nov 15 10:48:45 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1553000, "distortion": 1.2785031671524048, "distortion_nans": 0.0, "elbo": 2.3836932425498962, "elbo_filtered": 2.3836932425498962, "rate": 1.1051900757551194, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3224042985439302, "grad_norm": 44.38652420043945} +{"time": "Sun Nov 15 11:10:46 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1554000, "distortion": 1.2734152752161025, "distortion_nans": 0.0, "elbo": 2.3812886445522308, "elbo_filtered": 2.3812886445522308, "rate": 1.107873376250267, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3206968822479248, "grad_norm": 125.75442504882812} +{"time": "Sun Nov 15 11:32:47 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1555000, "distortion": 1.2729252952337264, "distortion_nans": 0.0, "elbo": 2.380371319770813, "elbo_filtered": 2.380371319770813, "rate": 1.1074460299015045, "rate_nans": 0.0, "skipped_updates": 1, "iter_time": 1.319480756521225, "grad_norm": 518.9920654296875} +{"time": "Sun Nov 15 11:54:51 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1556000, "distortion": 1.2749765733480454, "distortion_nans": 0.0, "elbo": 2.3844231626987455, "elbo_filtered": 2.3844231626987455, "rate": 1.1094465923309327, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3238186271190644, "grad_norm": 40.24611282348633} +{"time": "Sun Nov 15 12:16:54 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1557000, "distortion": 1.27588154733181, "distortion_nans": 0.0, "elbo": 2.38555890250206, "elbo_filtered": 2.38555890250206, "rate": 1.1096773607730865, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3213675320148468, "grad_norm": 20.604751586914062} +{"time": "Sun Nov 15 12:38:59 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1558000, "distortion": 1.2760969413518906, "distortion_nans": 0.0, "elbo": 2.3826527283191683, "elbo_filtered": 2.3826527283191683, "rate": 1.1065557901859284, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3244121341705322, "grad_norm": 26.921985626220703} +{"time": "Sun Nov 15 13:01:03 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 155, "step": 1559000, "distortion": 1.2758940507173537, "distortion_nans": 0.0, "elbo": 2.385688057899475, "elbo_filtered": 2.385688057899475, "rate": 1.109794000506401, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.322804859161377, "grad_norm": 21.582847595214844} +{"time": "Sun Nov 15 13:07:17 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 155, "step": 1559280, "n_batches": 39, "filtered_elbo": 2.3825598252125277, "distortion": 1.277816503475874, "elbo": 2.3825598252125277, "rate": 1.1047433431331928} +{"time": "Sun Nov 15 13:23:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 156, "step": 1560000, "distortion": 1.2783865478038787, "distortion_nans": 0.0, "elbo": 2.384748300552368, "elbo_filtered": 2.384748300552368, "rate": 1.1063617565631867, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3205700027942657, "grad_norm": 43.47957229614258} +{"time": "Sun Nov 15 13:23:10 2020", "message": "Saving model@ 1560000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 13:23:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1560000, "distortion": 1.2783773646354675, "distortion_nans": 0.0, "elbo": 2.3847215445041656, "elbo_filtered": 2.3847215445041656, "rate": 1.1063441836833954, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3206004083156586, "grad_norm": 43.47957229614258} +{"time": "Sun Nov 15 13:23:27 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1560000.png"} +{"time": "Sun Nov 15 13:45:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1561000, "distortion": 1.2738181680440903, "distortion_nans": 0.0, "elbo": 2.3806579990386965, "elbo_filtered": 2.3806579990386965, "rate": 1.1068398350477218, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3209751110076904, "grad_norm": 70.24889373779297} +{"time": "Sun Nov 15 14:07:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1562000, "distortion": 1.2760734121799469, "distortion_nans": 0.0, "elbo": 2.384468299150467, "elbo_filtered": 2.384468299150467, "rate": 1.108394894838333, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3190897924900056, "grad_norm": 22.20998764038086} +{"time": "Sun Nov 15 14:29:35 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1563000, "distortion": 1.27474973320961, "distortion_nans": 0.0, "elbo": 2.3848310706615448, "elbo_filtered": 2.3848310706615448, "rate": 1.110081335067749, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3251805448532104, "grad_norm": 20.871944427490234} +{"time": "Sun Nov 15 14:51:39 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1564000, "distortion": 1.274745752453804, "distortion_nans": 0.0, "elbo": 2.382813743829727, "elbo_filtered": 2.382813743829727, "rate": 1.1080679877996444, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3233655924797059, "grad_norm": 28.53388786315918} +{"time": "Sun Nov 15 15:13:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1565000, "distortion": 1.27703280544281, "distortion_nans": 0.0, "elbo": 2.383629882097244, "elbo_filtered": 2.383629882097244, "rate": 1.1065970828533174, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323445242881775, "grad_norm": 28.598651885986328} +{"time": "Sun Nov 15 15:35:38 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1566000, "distortion": 1.2788324645757676, "distortion_nans": 0.0, "elbo": 2.3835913383960725, "elbo_filtered": 2.3835913383960725, "rate": 1.1047588818073273, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3137580378055573, "grad_norm": 60.198883056640625} +{"time": "Sun Nov 15 15:57:34 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1567000, "distortion": 1.2788868420124053, "distortion_nans": 0.0, "elbo": 2.383338038921356, "elbo_filtered": 2.383338038921356, "rate": 1.1044512004852296, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3151167690753938, "grad_norm": 120.32221984863281} +{"time": "Sun Nov 15 16:19:33 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1568000, "distortion": 1.2730552932024002, "distortion_nans": 0.0, "elbo": 2.3815280339717866, "elbo_filtered": 2.3815280339717866, "rate": 1.1084727410078048, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3182337098121644, "grad_norm": 21.176162719726562} +{"time": "Sun Nov 15 16:41:26 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 156, "step": 1569000, "distortion": 1.2787638278007507, "distortion_nans": 0.0, "elbo": 2.3894317486286165, "elbo_filtered": 2.3894317486286165, "rate": 1.1106679272651672, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3115307796001434, "grad_norm": 35.72163391113281} +{"time": "Sun Nov 15 16:46:57 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 156, "step": 1569250, "n_batches": 39, "filtered_elbo": 2.381685250844711, "distortion": 1.2769856972572131, "elbo": 2.381685250844711, "rate": 1.1046995597007947} +{"time": "Sun Nov 15 17:03:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 157, "step": 1570000, "distortion": 1.274445454597473, "distortion_nans": 0.0, "elbo": 2.3801779613494873, "elbo_filtered": 2.3801779613494873, "rate": 1.1057325055599212, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.302768399000168, "grad_norm": 60.204463958740234} +{"time": "Sun Nov 15 17:03:16 2020", "message": "Saving model@ 1570000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 17:03:31 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1570000, "distortion": 1.2744128580093383, "distortion_nans": 0.0, "elbo": 2.3801402163505556, "elbo_filtered": 2.3801402163505556, "rate": 1.1057273570299149, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3029168951511383, "grad_norm": 60.204463958740234} +{"time": "Sun Nov 15 17:03:33 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1570000.png"} +{"time": "Sun Nov 15 17:25:25 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1571000, "distortion": 1.2777215213775635, "distortion_nans": 0.0, "elbo": 2.3824874482154845, "elbo_filtered": 2.3824874482154845, "rate": 1.1047659269571304, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.311333990097046, "grad_norm": 55.6642951965332} +{"time": "Sun Nov 15 17:47:22 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1572000, "distortion": 1.2781238652467728, "distortion_nans": 0.0, "elbo": 2.387694254875183, "elbo_filtered": 2.387694254875183, "rate": 1.1095703893899918, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3159793810844422, "grad_norm": 45.7606201171875} +{"time": "Sun Nov 15 18:09:27 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1573000, "distortion": 1.2726956819295883, "distortion_nans": 0.0, "elbo": 2.382705777168274, "elbo_filtered": 2.382705777168274, "rate": 1.1100101039409638, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3240441362857818, "grad_norm": 21.070161819458008} +{"time": "Sun Nov 15 18:31:20 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1574000, "distortion": 1.2753204704523087, "distortion_nans": 0.0, "elbo": 2.3847041962146758, "elbo_filtered": 2.3847041962146758, "rate": 1.1093837305307388, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3119070336818694, "grad_norm": 23.464332580566406} +{"time": "Sun Nov 15 18:53:20 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1575000, "distortion": 1.2752436946630479, "distortion_nans": 0.0, "elbo": 2.38147185754776, "elbo_filtered": 2.38147185754776, "rate": 1.1062281595468522, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3141867260932922, "grad_norm": 75.58551025390625} +{"time": "Sun Nov 15 19:15:20 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1576000, "distortion": 1.2781161429882049, "distortion_nans": 0.0, "elbo": 2.384645973920822, "elbo_filtered": 2.384645973920822, "rate": 1.106529841542244, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3195328373908997, "grad_norm": 74.65069580078125} +{"time": "Sun Nov 15 19:37:22 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1577000, "distortion": 1.2780457479953766, "distortion_nans": 0.0, "elbo": 2.3835048205852507, "elbo_filtered": 2.3835048205852507, "rate": 1.1054590755701066, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3209582073688506, "grad_norm": 45.5323371887207} +{"time": "Sun Nov 15 19:59:29 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1578000, "distortion": 1.2756662623882293, "distortion_nans": 0.0, "elbo": 2.383914736032486, "elbo_filtered": 2.383914736032486, "rate": 1.108248467206955, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3262720086574555, "grad_norm": 27.52852439880371} +{"time": "Sun Nov 15 20:21:35 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 157, "step": 1579000, "distortion": 1.2761939178705215, "distortion_nans": 0.0, "elbo": 2.384561468601227, "elbo_filtered": 2.384561468601227, "rate": 1.108367551445961, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3245726137161256, "grad_norm": 33.157188415527344} +{"time": "Sun Nov 15 20:26:30 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 157, "step": 1579220, "n_batches": 39, "filtered_elbo": 2.3825646363771877, "distortion": 1.2776117783326368, "elbo": 2.3825646363771877, "rate": 1.1049528702711449} +{"time": "Sun Nov 15 20:43:40 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 158, "step": 1580000, "distortion": 1.27680341899395, "distortion_nans": 0.0, "elbo": 2.383825068473816, "elbo_filtered": 2.383825068473816, "rate": 1.1070216468572616, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318033435344696, "grad_norm": 106.08421325683594} +{"time": "Sun Nov 15 20:43:40 2020", "message": "Saving model@ 1580000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Sun Nov 15 20:43:55 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1580000, "distortion": 1.276922539949417, "distortion_nans": 0.0, "elbo": 2.3839147183895113, "elbo_filtered": 2.3839147183895113, "rate": 1.1069921754598617, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.318106448173523, "grad_norm": 106.08421325683594} +{"time": "Sun Nov 15 20:43:56 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1580000.png"} +{"time": "Sun Nov 15 21:05:52 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1581000, "distortion": 1.2738780994415284, "distortion_nans": 0.0, "elbo": 2.3832765128612516, "elbo_filtered": 2.3832765128612516, "rate": 1.1093984233140946, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3146248228549957, "grad_norm": 22.639055252075195} +{"time": "Sun Nov 15 21:27:57 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1582000, "distortion": 1.273894619822502, "distortion_nans": 0.0, "elbo": 2.3837943964004515, "elbo_filtered": 2.3837943964004515, "rate": 1.1098997795581818, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.323461140871048, "grad_norm": 30.355682373046875} +{"time": "Sun Nov 15 21:49:49 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1583000, "distortion": 1.2760607087612152, "distortion_nans": 0.0, "elbo": 2.3832411341667177, "elbo_filtered": 2.3832411341667177, "rate": 1.1071804271936416, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3115809397697449, "grad_norm": 83.34394073486328} +{"time": "Sun Nov 15 22:11:23 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1584000, "distortion": 1.2778969798088073, "distortion_nans": 0.0, "elbo": 2.383575891971588, "elbo_filtered": 2.383575891971588, "rate": 1.1056789177656174, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.2933133823871612, "grad_norm": 97.6307144165039} +{"time": "Sun Nov 15 22:33:10 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1585000, "distortion": 1.2756664643287658, "distortion_nans": 0.0, "elbo": 2.3844184594154356, "elbo_filtered": 2.3844184594154356, "rate": 1.108751997947693, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3060410439968109, "grad_norm": 30.27864646911621} +{"time": "Sun Nov 15 22:55:04 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1586000, "distortion": 1.2735624930858611, "distortion_nans": 0.0, "elbo": 2.382563888311386, "elbo_filtered": 2.382563888311386, "rate": 1.1090013914108277, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3132976133823395, "grad_norm": 21.01503562927246} +{"time": "Sun Nov 15 23:17:07 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1587000, "distortion": 1.2747177387475968, "distortion_nans": 0.0, "elbo": 2.3845728130340578, "elbo_filtered": 2.3845728130340578, "rate": 1.1098550837039947, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3219120907783508, "grad_norm": 21.276121139526367} +{"time": "Sun Nov 15 23:38:56 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1588000, "distortion": 1.2733001061677933, "distortion_nans": 0.0, "elbo": 2.384084112405777, "elbo_filtered": 2.384084112405777, "rate": 1.1107840093374253, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.308357933998108, "grad_norm": 21.413721084594727} +{"time": "Mon Nov 16 00:00:49 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 158, "step": 1589000, "distortion": 1.271452742934227, "distortion_nans": 0.0, "elbo": 2.3813826460838317, "elbo_filtered": 2.3813826460838317, "rate": 1.1099299093484878, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3112412288188935, "grad_norm": 49.661285400390625} +{"time": "Mon Nov 16 00:05:05 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 158, "step": 1589190, "n_batches": 39, "filtered_elbo": 2.3821293207315297, "distortion": 1.2741067501214833, "elbo": 2.3821293207315297, "rate": 1.1080225431002104} +{"time": "Mon Nov 16 00:22:44 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 159, "step": 1590000, "distortion": 1.2752656285762787, "distortion_nans": 0.0, "elbo": 2.3825386991500856, "elbo_filtered": 2.3825386991500856, "rate": 1.1072730666399002, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3086821806430817, "grad_norm": 24.773025512695312} +{"time": "Mon Nov 16 00:22:44 2020", "message": "Saving model@ 1590000 to /root/vaes/oct11-i64-re1-lr05/latest"} +{"time": "Mon Nov 16 00:22:59 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1590000, "distortion": 1.2752948701381683, "distortion_nans": 0.0, "elbo": 2.382584689617157, "elbo_filtered": 2.382584689617157, "rate": 1.1072898151874542, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3086677696704865, "grad_norm": 24.773025512695312} +{"time": "Mon Nov 16 00:23:01 2020", "message": "printing samples to /root/vaes/oct11-i64-re1-lr05/samples-1590000.png"} +{"time": "Mon Nov 16 00:44:59 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1591000, "distortion": 1.2755418920516968, "distortion_nans": 0.0, "elbo": 2.384043969631195, "elbo_filtered": 2.384043969631195, "rate": 1.1085020837783814, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3168731966018676, "grad_norm": 20.385011672973633} +{"time": "Mon Nov 16 01:06:55 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1592000, "distortion": 1.274278483390808, "distortion_nans": 0.0, "elbo": 2.3830591843128204, "elbo_filtered": 2.3830591843128204, "rate": 1.1087807055711747, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3155321238040925, "grad_norm": 21.48441505432129} +{"time": "Mon Nov 16 01:28:50 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1593000, "distortion": 1.2736258982419968, "distortion_nans": 0.0, "elbo": 2.3818965799808502, "elbo_filtered": 2.3818965799808502, "rate": 1.1082706823349, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3135502214431762, "grad_norm": 22.010957717895508} +{"time": "Mon Nov 16 01:50:40 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1594000, "distortion": 1.276010708451271, "distortion_nans": 0.0, "elbo": 2.384713913679123, "elbo_filtered": 2.384713913679123, "rate": 1.1087032073736192, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3095307528972626, "grad_norm": 23.45583724975586} +{"time": "Mon Nov 16 02:12:24 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1595000, "distortion": 1.27133917427063, "distortion_nans": 0.0, "elbo": 2.3814001705646515, "elbo_filtered": 2.3814001705646515, "rate": 1.1100610010623932, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3028911471366882, "grad_norm": 23.913188934326172} +{"time": "Mon Nov 16 02:34:16 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1596000, "distortion": 1.2746312975883485, "distortion_nans": 0.0, "elbo": 2.384935432910919, "elbo_filtered": 2.384935432910919, "rate": 1.1103041368722915, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3108242809772492, "grad_norm": 34.11654281616211} +{"time": "Mon Nov 16 02:56:15 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1597000, "distortion": 1.2767363679409027, "distortion_nans": 0.0, "elbo": 2.385619741678238, "elbo_filtered": 2.385619741678238, "rate": 1.108883375585079, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3182810618877412, "grad_norm": 121.3915786743164} +{"time": "Mon Nov 16 03:18:05 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1598000, "distortion": 1.2734732202291488, "distortion_nans": 0.0, "elbo": 2.382997249364853, "elbo_filtered": 2.382997249364853, "rate": 1.1095240296125413, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.309546692609787, "grad_norm": 24.03042221069336} +{"time": "Mon Nov 16 03:39:59 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "lr": 5e-05, "epoch": 159, "step": 1599000, "distortion": 1.2735858345031739, "distortion_nans": 0.0, "elbo": 2.38196715426445, "elbo_filtered": 2.38196715426445, "rate": 1.1083813154697417, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3125252985954285, "grad_norm": 48.67327880859375} +{"time": "Mon Nov 16 03:43:33 2020", "model": "oct11-i64-re1-lr05", "type": "eval_loss", "epoch": 159, "step": 1599160, "n_batches": 39, "filtered_elbo": 2.3827869708721456, "distortion": 1.2761272681065094, "elbo": 2.3827869708721456, "rate": 1.1066596874823937} +{"time": "Mon Nov 16 04:01:58 2020", "model": "oct11-i64-re1-lr05", "type": "train_loss", "epoch": 160, "step": 1600000, "distortion": 1.2759663579463958, "distortion_nans": 0.0, "elbo": 2.383661519050598, "elbo_filtered": 2.383661519050598, "rate": 1.1076951621770859, "rate_nans": 0.0, "skipped_updates": 0, "iter_time": 1.3120145084857942, "grad_norm": 23.525554656982422} +{"time": "Mon Nov 16 04:01:58 2020", "message": "Saving model@ 1600000 to /root/vaes/oct11-i64-re1-lr05/latest"} diff --git a/vdvae/model_utils.py b/vdvae/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2f5d9194b0e7de5feef817bdd42f514e742db4ea --- /dev/null +++ b/vdvae/model_utils.py @@ -0,0 +1,102 @@ +import torch +import numpy as np +#from mpi4py import MPI +import socket +import argparse +import os +import json +import subprocess +from hps import Hyperparams, parse_args_and_update_hparams, add_vae_arguments +from utils import (logger, + local_mpi_rank, + mpi_size, + maybe_download, + mpi_rank) +from data import mkdir_p +from contextlib import contextmanager +import torch.distributed as dist +#from apex.optimizers import FusedAdam as AdamW +from vae import VAE +from torch.nn.parallel.distributed import DistributedDataParallel +from train_helpers import restore_params + +def set_up_hyperparams(s=None): + H = Hyperparams() + parser = argparse.ArgumentParser() + parser = add_vae_arguments(parser) + parse_args_and_update_hparams(H, parser, s=s) + setup_mpi(H) + setup_save_dirs(H) + logprint = logger(H.logdir) + for i, k in enumerate(sorted(H)): + logprint(type='hparam', key=k, value=H[k]) + np.random.seed(H.seed) + torch.manual_seed(H.seed) + torch.cuda.manual_seed(H.seed) + logprint('training model', H.desc, 'on', H.dataset) + return H, logprint + +def set_up_data(H): + shift_loss = -127.5 + scale_loss = 1. / 127.5 + + #trX, vaX, teX = imagenet64(H.data_root) + H.image_size = 64 + H.image_channels = 3 + shift = -115.92961967 + scale = 1. / 69.37404 + + #if H.test_eval: + # print('DOING TEST') + # eval_dataset = teX + #else: + # eval_dataset = vaX + + shift = torch.tensor([shift]).cuda().view(1, 1, 1, 1) + scale = torch.tensor([scale]).cuda().view(1, 1, 1, 1) + shift_loss = torch.tensor([shift_loss]).cuda().view(1, 1, 1, 1) + scale_loss = torch.tensor([scale_loss]).cuda().view(1, 1, 1, 1) + + #train_data = TensorDataset(torch.as_tensor(trX)) + #valid_data = TensorDataset(torch.as_tensor(eval_dataset)) + #untranspose = False + + def preprocess_func(x): + nonlocal shift + nonlocal scale + nonlocal shift_loss + nonlocal scale_loss + 'takes in a data example and returns the preprocessed input' + 'as well as the input processed for the loss' + #untranspose = False + #if untranspose: + # x[0] = x[0].permute(0, 2, 3, 1) + inp = x.cuda(non_blocking=True).float() + out = inp.clone() + inp.add_(shift).mul_(scale) + out.add_(shift_loss).mul_(scale_loss) + return inp, out + + return H, preprocess_func + +def load_vaes(H, logprint=None): + + ema_vae = VAE(H) + if H.restore_ema_path: + print(f'Restoring ema vae from {H.restore_ema_path}') + restore_params(ema_vae, H.restore_ema_path, map_cpu=True, local_rank=H.local_rank, mpi_size=H.mpi_size) + else: + ema_vae.load_state_dict(vae.state_dict()) + ema_vae.requires_grad_(False) + ema_vae = ema_vae.cuda(H.local_rank) + + #vae = DistributedDataParallel(vae, device_ids=[H.local_rank], output_device=H.local_rank) + + #if len(list(vae.named_parameters())) != len(list(vae.parameters())): + # raise ValueError('Some params are not named. Please name all params.') + #total_params = 0 + #for name, p in vae.named_parameters(): + # total_params += np.prod(p.shape) + + #print(total_params=total_params, readable=f'{total_params:,}') + return ema_vae \ No newline at end of file diff --git a/vdvae/setup_cifar10.sh b/vdvae/setup_cifar10.sh new file mode 100644 index 0000000000000000000000000000000000000000..42b6c54229d327a033355d57d2476033ab15d860 --- /dev/null +++ b/vdvae/setup_cifar10.sh @@ -0,0 +1,2 @@ +wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz +tar -xf cifar-10-python.tar.gz diff --git a/vdvae/setup_ffhq1024.sh b/vdvae/setup_ffhq1024.sh new file mode 100644 index 0000000000000000000000000000000000000000..511066400a6b89180c008c65e88b86868265ac48 --- /dev/null +++ b/vdvae/setup_ffhq1024.sh @@ -0,0 +1,13 @@ +# the first argument to this script should be the path to the ffhq_images1024x1024 folder +# the same path should be provided as the `data_root` argument to train.py +cd $1 +mkdir train +mkdir train/0 +mkdir valid +mkdir valid/0 +for i in $(seq -f "%05g" 0 64999); do + mv $i.png train/0 +done +for i in $(seq -f "%05g" 65000 69999); do + mv $i.png valid/0 +done diff --git a/vdvae/setup_ffhq256.sh b/vdvae/setup_ffhq256.sh new file mode 100644 index 0000000000000000000000000000000000000000..454a81f669207cda28be3905dba50be54c65fafa --- /dev/null +++ b/vdvae/setup_ffhq256.sh @@ -0,0 +1,12 @@ +# we provide a copy of ffhq-256 for convenience, downsampled using the same function as NVAE (https://github.com/NVlabs/NVAE) (personal communication with author) + +# Resizing function is this one, with the default second argument and size=256 +# https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.Resize + +# 5-bit precision is calculated using the following lines, with num_bits=5 and for an x in [0, 1] +# x = torch.floor(x * 255 / 2 ** (8 - num_bits)) +# x /= (2 ** num_bits - 1) + +# the DMOL loss should also be adjusted to have 32 buckets instead of 256 (this code, or NVAE, can be used as reference) + +wget https://openaipublic.blob.core.windows.net/very-deep-vaes-assets/vdvae-assets/ffhq-256.npy \ No newline at end of file diff --git a/vdvae/setup_imagenet.sh b/vdvae/setup_imagenet.sh new file mode 100644 index 0000000000000000000000000000000000000000..fec4223129fb3cd80100e08d8542ac5baea95a02 --- /dev/null +++ b/vdvae/setup_imagenet.sh @@ -0,0 +1,28 @@ +if [ "$1" == "imagenet32" ]; then + +echo "downloading imagenet32" +wget http://www.image-net.org/small/train_32x32.tar +wget http://www.image-net.org/small/valid_32x32.tar +tar -xvf train_32x32.tar +tar -xvf valid_32x32.tar +python files_to_npy.py train_32x32/ imagenet32-train.npy +python files_to_npy.py valid_32x32/ imagenet32-valid.npy + +elif [ "$1" == "imagenet64" ]; then + +echo "downloading imagenet64" +wget http://www.image-net.org/small/train_64x64.tar +wget http://www.image-net.org/small/valid_64x64.tar +tar -xvf train_64x64.tar +tar -xvf valid_64x64.tar +python files_to_npy.py train_64x64/ imagenet64-train.npy +python files_to_npy.py valid_64x64/ imagenet64-valid.npy + +else + +echo "please pass the string imagenet32 or imagenet64 as an argument" + +fi + + + diff --git a/vdvae/train.py b/vdvae/train.py new file mode 100644 index 0000000000000000000000000000000000000000..0540cb0b94ea049c8abfdb268d5122f6ebcf09d8 --- /dev/null +++ b/vdvae/train.py @@ -0,0 +1,136 @@ +import numpy as np +import imageio +import os +import time +import torch +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from data import set_up_data +from utils import get_cpu_stats_over_ranks +from train_helpers import set_up_hyperparams, load_vaes, load_opt, accumulate_stats, save_model, update_ema + + +def training_step(H, data_input, target, vae, ema_vae, optimizer, iterate): + t0 = time.time() + vae.zero_grad() + stats = vae.forward(data_input, target) + stats['elbo'].backward() + grad_norm = torch.nn.utils.clip_grad_norm_(vae.parameters(), H.grad_clip).item() + distortion_nans = torch.isnan(stats['distortion']).sum() + rate_nans = torch.isnan(stats['rate']).sum() + stats.update(dict(rate_nans=0 if rate_nans == 0 else 1, distortion_nans=0 if distortion_nans == 0 else 1)) + stats = get_cpu_stats_over_ranks(stats) + + skipped_updates = 1 + # only update if no rank has a nan and if the grad norm is below a specific threshold + if stats['distortion_nans'] == 0 and stats['rate_nans'] == 0 and (H.skip_threshold == -1 or grad_norm < H.skip_threshold): + optimizer.step() + skipped_updates = 0 + update_ema(vae, ema_vae, H.ema_rate) + + t1 = time.time() + stats.update(skipped_updates=skipped_updates, iter_time=t1 - t0, grad_norm=grad_norm) + return stats + + +def eval_step(data_input, target, ema_vae): + with torch.no_grad(): + stats = ema_vae.forward(data_input, target) + stats = get_cpu_stats_over_ranks(stats) + return stats + + +def get_sample_for_visualization(data, preprocess_fn, num, dataset): + for x in DataLoader(data, batch_size=num): + break + orig_image = (x[0] * 255.0).to(torch.uint8).permute(0, 2, 3, 1) if dataset == 'ffhq_1024' else x[0] + preprocessed = preprocess_fn(x)[0] + return orig_image, preprocessed + + +def train_loop(H, data_train, data_valid, preprocess_fn, vae, ema_vae, logprint): + optimizer, scheduler, cur_eval_loss, iterate, starting_epoch = load_opt(H, vae, logprint) + train_sampler = DistributedSampler(data_train, num_replicas=H.mpi_size, rank=H.rank) + viz_batch_original, viz_batch_processed = get_sample_for_visualization(data_valid, preprocess_fn, H.num_images_visualize, H.dataset) + early_evals = set([1] + [2 ** exp for exp in range(3, 14)]) + stats = [] + iters_since_starting = 0 + H.ema_rate = torch.as_tensor(H.ema_rate).cuda() + for epoch in range(starting_epoch, H.num_epochs): + train_sampler.set_epoch(epoch) + for x in DataLoader(data_train, batch_size=H.n_batch, drop_last=True, pin_memory=True, sampler=train_sampler): + data_input, target = preprocess_fn(x) + training_stats = training_step(H, data_input, target, vae, ema_vae, optimizer, iterate) + stats.append(training_stats) + scheduler.step() + if iterate % H.iters_per_print == 0 or iters_since_starting in early_evals: + logprint(model=H.desc, type='train_loss', lr=scheduler.get_last_lr()[0], epoch=epoch, step=iterate, **accumulate_stats(stats, H.iters_per_print)) + + if iterate % H.iters_per_images == 0 or (iters_since_starting in early_evals and H.dataset != 'ffhq_1024') and H.rank == 0: + write_images(H, ema_vae, viz_batch_original, viz_batch_processed, f'{H.save_dir}/samples-{iterate}.png', logprint) + + iterate += 1 + iters_since_starting += 1 + if iterate % H.iters_per_save == 0 and H.rank == 0: + if np.isfinite(stats[-1]['elbo']): + logprint(model=H.desc, type='train_loss', epoch=epoch, step=iterate, **accumulate_stats(stats, H.iters_per_print)) + fp = os.path.join(H.save_dir, 'latest') + logprint(f'Saving model@ {iterate} to {fp}') + save_model(fp, vae, ema_vae, optimizer, H) + + if iterate % H.iters_per_ckpt == 0 and H.rank == 0: + save_model(os.path.join(H.save_dir, f'iter-{iterate}'), vae, ema_vae, optimizer, H) + + if epoch % H.epochs_per_eval == 0: + valid_stats = evaluate(H, ema_vae, data_valid, preprocess_fn) + logprint(model=H.desc, type='eval_loss', epoch=epoch, step=iterate, **valid_stats) + + +def evaluate(H, ema_vae, data_valid, preprocess_fn): + stats_valid = [] + valid_sampler = DistributedSampler(data_valid, num_replicas=H.mpi_size, rank=H.rank) + for x in DataLoader(data_valid, batch_size=H.n_batch, drop_last=True, pin_memory=True, sampler=valid_sampler): + data_input, target = preprocess_fn(x) + stats_valid.append(eval_step(data_input, target, ema_vae)) + vals = [a['elbo'] for a in stats_valid] + finites = np.array(vals)[np.isfinite(vals)] + stats = dict(n_batches=len(vals), filtered_elbo=np.mean(finites), **{k: np.mean([a[k] for a in stats_valid]) for k in stats_valid[-1]}) + return stats + + +def write_images(H, ema_vae, viz_batch_original, viz_batch_processed, fname, logprint): + zs = [s['z'].cuda() for s in ema_vae.forward_get_latents(viz_batch_processed)] + batches = [viz_batch_original.numpy()] + mb = viz_batch_processed.shape[0] + lv_points = np.floor(np.linspace(0, 1, H.num_variables_visualize + 2) * len(zs)).astype(int)[1:-1] + for i in lv_points: + batches.append(ema_vae.forward_samples_set_latents(mb, zs[:i], t=0.1)) + for t in [1.0, 0.9, 0.8, 0.7][:H.num_temperatures_visualize]: + batches.append(ema_vae.forward_uncond_samples(mb, t=t)) + n_rows = len(batches) + im = np.concatenate(batches, axis=0).reshape((n_rows, mb, *viz_batch_processed.shape[1:])).transpose([0, 2, 1, 3, 4]).reshape([n_rows * viz_batch_processed.shape[1], mb * viz_batch_processed.shape[2], 3]) + logprint(f'printing samples to {fname}') + imageio.imwrite(fname, im) + + +def run_test_eval(H, ema_vae, data_test, preprocess_fn, logprint): + print('evaluating') + stats = evaluate(H, ema_vae, data_test, preprocess_fn) + print('test results') + for k in stats: + print(k, stats[k]) + logprint(type='test_loss', **stats) + + +def main(): + H, logprint = set_up_hyperparams() + H, data_train, data_valid_or_test, preprocess_fn = set_up_data(H) + vae, ema_vae = load_vaes(H, logprint) + if H.test_eval: + run_test_eval(H, ema_vae, data_valid_or_test, preprocess_fn, logprint) + else: + train_loop(H, data_train, data_valid_or_test, preprocess_fn, vae, ema_vae, logprint) + + +if __name__ == "__main__": + main() diff --git a/vdvae/train_helpers.py b/vdvae/train_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ed12cf3003deae9dc07aa8173a941c0f230526 --- /dev/null +++ b/vdvae/train_helpers.py @@ -0,0 +1,190 @@ +import torch +import numpy as np +#from mpi4py import MPI +import socket +import argparse +import os +import json +import subprocess +from hps import Hyperparams, parse_args_and_update_hparams, add_vae_arguments +from utils import (logger, + local_mpi_rank, + mpi_size, + maybe_download, + mpi_rank) +from data import mkdir_p +from contextlib import contextmanager +import torch.distributed as dist +#from apex.optimizers import FusedAdam as AdamW +from vae import VAE +from torch.nn.parallel.distributed import DistributedDataParallel + + +def update_ema(vae, ema_vae, ema_rate): + for p1, p2 in zip(vae.parameters(), ema_vae.parameters()): + p2.data.mul_(ema_rate) + p2.data.add_(p1.data * (1 - ema_rate)) + + +def save_model(path, vae, ema_vae, optimizer, H): + torch.save(vae.state_dict(), f'{path}-model.th') + torch.save(ema_vae.state_dict(), f'{path}-model-ema.th') + torch.save(optimizer.state_dict(), f'{path}-opt.th') + from_log = os.path.join(H.save_dir, 'log.jsonl') + to_log = f'{os.path.dirname(path)}/{os.path.basename(path)}-log.jsonl' + subprocess.check_output(['cp', from_log, to_log]) + + +def accumulate_stats(stats, frequency): + z = {} + for k in stats[-1]: + if k in ['distortion_nans', 'rate_nans', 'skipped_updates', 'gcskip']: + z[k] = np.sum([a[k] for a in stats[-frequency:]]) + elif k == 'grad_norm': + vals = [a[k] for a in stats[-frequency:]] + finites = np.array(vals)[np.isfinite(vals)] + if len(finites) == 0: + z[k] = 0.0 + else: + z[k] = np.max(finites) + elif k == 'elbo': + vals = [a[k] for a in stats[-frequency:]] + finites = np.array(vals)[np.isfinite(vals)] + z['elbo'] = np.mean(vals) + z['elbo_filtered'] = np.mean(finites) + elif k == 'iter_time': + z[k] = stats[-1][k] if len(stats) < frequency else np.mean([a[k] for a in stats[-frequency:]]) + else: + z[k] = np.mean([a[k] for a in stats[-frequency:]]) + return z + + +def linear_warmup(warmup_iters): + def f(iteration): + return 1.0 if iteration > warmup_iters else iteration / warmup_iters + return f + + +def setup_mpi(H): + H.mpi_size = mpi_size() + H.local_rank = local_mpi_rank() + H.rank = mpi_rank() + os.environ["RANK"] = str(H.rank) + os.environ["WORLD_SIZE"] = str(H.mpi_size) + os.environ["MASTER_PORT"] = str(H.port) + # os.environ["NCCL_LL_THRESHOLD"] = "0" + os.environ["MASTER_ADDR"] = MPI.COMM_WORLD.bcast(socket.gethostname(), root=0) + torch.cuda.set_device(H.local_rank) + dist.init_process_group(backend='nccl', init_method=f"env://") + + +def distributed_maybe_download(path, local_rank, mpi_size): + if not path.startswith('gs://'): + return path + filename = path[5:].replace('/', '-') + with first_rank_first(local_rank, mpi_size): + fp = maybe_download(path, filename) + return fp + + +@contextmanager +def first_rank_first(local_rank, mpi_size): + if mpi_size > 1 and local_rank > 0: + dist.barrier() + + try: + yield + finally: + if mpi_size > 1 and local_rank == 0: + dist.barrier() + + +def setup_save_dirs(H): + H.save_dir = os.path.join(H.save_dir, H.desc) + mkdir_p(H.save_dir) + H.logdir = os.path.join(H.save_dir, 'log') + + +def set_up_hyperparams(s=None): + H = Hyperparams() + parser = argparse.ArgumentParser() + parser = add_vae_arguments(parser) + parse_args_and_update_hparams(H, parser, s=s) + setup_mpi(H) + setup_save_dirs(H) + logprint = logger(H.logdir) + for i, k in enumerate(sorted(H)): + logprint(type='hparam', key=k, value=H[k]) + np.random.seed(H.seed) + torch.manual_seed(H.seed) + torch.cuda.manual_seed(H.seed) + logprint('training model', H.desc, 'on', H.dataset) + return H, logprint + + +def restore_params(model, path, local_rank, mpi_size, map_ddp=True, map_cpu=False): + state_dict = torch.load(distributed_maybe_download(path, local_rank, mpi_size), map_location='cpu' if map_cpu else None) + if map_ddp: + new_state_dict = {} + l = len('module.') + for k in state_dict: + if k.startswith('module.'): + new_state_dict[k[l:]] = state_dict[k] + else: + new_state_dict[k] = state_dict[k] + state_dict = new_state_dict + model.load_state_dict(state_dict) + + +def restore_log(path, local_rank, mpi_size): + loaded = [json.loads(l) for l in open(distributed_maybe_download(path, local_rank, mpi_size))] + try: + cur_eval_loss = min([z['elbo'] for z in loaded if 'type' in z and z['type'] == 'eval_loss']) + except ValueError: + cur_eval_loss = float('inf') + starting_epoch = max([z['epoch'] for z in loaded if 'type' in z and z['type'] == 'train_loss']) + iterate = max([z['step'] for z in loaded if 'type' in z and z['type'] == 'train_loss']) + return cur_eval_loss, iterate, starting_epoch + + +def load_vaes(H, logprint): + vae = VAE(H) + if H.restore_path: + logprint(f'Restoring vae from {H.restore_path}') + restore_params(vae, H.restore_path, map_cpu=True, local_rank=H.local_rank, mpi_size=H.mpi_size) + + ema_vae = VAE(H) + if H.restore_ema_path: + logprint(f'Restoring ema vae from {H.restore_ema_path}') + restore_params(ema_vae, H.restore_ema_path, map_cpu=True, local_rank=H.local_rank, mpi_size=H.mpi_size) + else: + ema_vae.load_state_dict(vae.state_dict()) + ema_vae.requires_grad_(False) + + vae = vae.cuda(H.local_rank) + ema_vae = ema_vae.cuda(H.local_rank) + + vae = DistributedDataParallel(vae, device_ids=[H.local_rank], output_device=H.local_rank) + + if len(list(vae.named_parameters())) != len(list(vae.parameters())): + raise ValueError('Some params are not named. Please name all params.') + total_params = 0 + for name, p in vae.named_parameters(): + total_params += np.prod(p.shape) + logprint(total_params=total_params, readable=f'{total_params:,}') + return vae, ema_vae + + +def load_opt(H, vae, logprint): + optimizer = AdamW(vae.parameters(), weight_decay=H.wd, lr=H.lr, betas=(H.adam_beta1, H.adam_beta2)) + scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=linear_warmup(H.warmup_iters)) + + if H.restore_optimizer_path: + optimizer.load_state_dict( + torch.load(distributed_maybe_download(H.restore_optimizer_path, H.local_rank, H.mpi_size), map_location='cpu')) + if H.restore_log_path: + cur_eval_loss, iterate, starting_epoch = restore_log(H.restore_log_path, H.local_rank, H.mpi_size) + else: + cur_eval_loss, iterate, starting_epoch = float('inf'), 0, 0 + logprint('starting at epoch', starting_epoch, 'iterate', iterate, 'eval loss', cur_eval_loss) + return optimizer, scheduler, cur_eval_loss, iterate, starting_epoch diff --git a/vdvae/utils.py b/vdvae/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ae1544b8c817ccade236f082305065bdae538e62 --- /dev/null +++ b/vdvae/utils.py @@ -0,0 +1,140 @@ +#from mpi4py import MPI +import os +import json +import tempfile +import numpy as np +import torch +import time +import subprocess +import torch.distributed as dist + + +def allreduce(x, average): + if mpi_size() > 1: + dist.all_reduce(x, dist.ReduceOp.SUM) + return x / mpi_size() if average else x + + +def get_cpu_stats_over_ranks(stat_dict): + keys = sorted(stat_dict.keys()) + allreduced = allreduce(torch.stack([torch.as_tensor(stat_dict[k]).detach().cuda().float() for k in keys]), average=True).cpu() + return {k: allreduced[i].item() for (i, k) in enumerate(keys)} + + +class Hyperparams(dict): + def __getattr__(self, attr): + try: + return self[attr] + except KeyError: + return None + + def __setattr__(self, attr, value): + self[attr] = value + + +def logger(log_prefix): + 'Prints the arguments out to stdout, .txt, and .jsonl files' + + jsonl_path = f'{log_prefix}.jsonl' + txt_path = f'{log_prefix}.txt' + + def log(*args, pprint=False, **kwargs): + if mpi_rank() != 0: + return + t = time.ctime() + argdict = {'time': t} + if len(args) > 0: + argdict['message'] = ' '.join([str(x) for x in args]) + argdict.update(kwargs) + + txt_str = [] + args_iter = sorted(argdict) if pprint else argdict + for k in args_iter: + val = argdict[k] + if isinstance(val, np.ndarray): + val = val.tolist() + elif isinstance(val, np.integer): + val = int(val) + elif isinstance(val, np.floating): + val = float(val) + argdict[k] = val + if isinstance(val, float): + val = f'{val:.5f}' + txt_str.append(f'{k}: {val}') + txt_str = ', '.join(txt_str) + + if pprint: + json_str = json.dumps(argdict, sort_keys=True) + txt_str = json.dumps(argdict, sort_keys=True, indent=4) + else: + json_str = json.dumps(argdict) + + print(txt_str, flush=True) + + with open(txt_path, "a+") as f: + print(txt_str, file=f, flush=True) + with open(jsonl_path, "a+") as f: + print(json_str, file=f, flush=True) + + return log + + +def maybe_download(path, filename=None): + '''If a path is a gsutil path, download it and return the local link, + otherwise return link''' + if not path.startswith('gs://'): + return path + if filename: + local_dest = f'/tmp/' + out_path = f'/tmp/{filename}' + if os.path.isfile(out_path): + return out_path + subprocess.check_output(['gsutil', '-m', 'cp', '-R', path, out_path]) + return out_path + else: + local_dest = tempfile.mkstemp()[1] + subprocess.check_output(['gsutil', '-m', 'cp', path, local_dest]) + return local_dest + + +def tile_images(images, d1=4, d2=4, border=1): + id1, id2, c = images[0].shape + out = np.ones([d1 * id1 + border * (d1 + 1), + d2 * id2 + border * (d2 + 1), + c], dtype=np.uint8) + out *= 255 + if len(images) != d1 * d2: + raise ValueError('Wrong num of images') + for imgnum, im in enumerate(images): + num_d1 = imgnum // d2 + num_d2 = imgnum % d2 + start_d1 = num_d1 * id1 + border * (num_d1 + 1) + start_d2 = num_d2 * id2 + border * (num_d2 + 1) + out[start_d1:start_d1 + id1, start_d2:start_d2 + id2, :] = im + return out + + +def mpi_size(): + return MPI.COMM_WORLD.Get_size() + + +def mpi_rank(): + return MPI.COMM_WORLD.Get_rank() + + +def num_nodes(): + nn = mpi_size() + if nn % 8 == 0: + return nn // 8 + return nn // 8 + 1 + + +def gpus_per_node(): + size = mpi_size() + if size > 1: + return max(size // num_nodes(), 1) + return 1 + + +def local_mpi_rank(): + return mpi_rank() % gpus_per_node() diff --git a/vdvae/vae.py b/vdvae/vae.py new file mode 100644 index 0000000000000000000000000000000000000000..97b08874d1c436fc9fd7a7c3b32ae44cf4656330 --- /dev/null +++ b/vdvae/vae.py @@ -0,0 +1,252 @@ +import torch +from torch import nn +from torch.nn import functional as F +from vae_helpers import HModule, get_1x1, get_3x3, DmolNet, draw_gaussian_diag_samples, gaussian_analytical_kl +from collections import defaultdict +import numpy as np +import itertools + + +class Block(nn.Module): + def __init__(self, in_width, middle_width, out_width, down_rate=None, residual=False, use_3x3=True, zero_last=False): + super().__init__() + self.down_rate = down_rate + self.residual = residual + self.c1 = get_1x1(in_width, middle_width) + self.c2 = get_3x3(middle_width, middle_width) if use_3x3 else get_1x1(middle_width, middle_width) + self.c3 = get_3x3(middle_width, middle_width) if use_3x3 else get_1x1(middle_width, middle_width) + self.c4 = get_1x1(middle_width, out_width, zero_weights=zero_last) + + def forward(self, x): + xhat = self.c1(F.gelu(x)) + xhat = self.c2(F.gelu(xhat)) + xhat = self.c3(F.gelu(xhat)) + xhat = self.c4(F.gelu(xhat)) + out = x + xhat if self.residual else xhat + if self.down_rate is not None: + out = F.avg_pool2d(out, kernel_size=self.down_rate, stride=self.down_rate) + return out + + +def parse_layer_string(s): + layers = [] + for ss in s.split(','): + if 'x' in ss: + res, num = ss.split('x') + count = int(num) + layers += [(int(res), None) for _ in range(count)] + elif 'm' in ss: + res, mixin = [int(a) for a in ss.split('m')] + layers.append((res, mixin)) + elif 'd' in ss: + res, down_rate = [int(a) for a in ss.split('d')] + layers.append((res, down_rate)) + else: + res = int(ss) + layers.append((res, None)) + return layers + + +def pad_channels(t, width): + d1, d2, d3, d4 = t.shape + empty = torch.zeros(d1, width, d3, d4, device=t.device) + empty[:, :d2, :, :] = t + return empty + + +def get_width_settings(width, s): + mapping = defaultdict(lambda: width) + if s: + s = s.split(',') + for ss in s: + k, v = ss.split(':') + mapping[int(k)] = int(v) + return mapping + + +class Encoder(HModule): + def build(self): + H = self.H + self.in_conv = get_3x3(H.image_channels, H.width) + self.widths = get_width_settings(H.width, H.custom_width_str) + enc_blocks = [] + blockstr = parse_layer_string(H.enc_blocks) + for res, down_rate in blockstr: + use_3x3 = res > 2 # Don't use 3x3s for 1x1, 2x2 patches + enc_blocks.append(Block(self.widths[res], int(self.widths[res] * H.bottleneck_multiple), self.widths[res], down_rate=down_rate, residual=True, use_3x3=use_3x3)) + n_blocks = len(blockstr) + for b in enc_blocks: + b.c4.weight.data *= np.sqrt(1 / n_blocks) + self.enc_blocks = nn.ModuleList(enc_blocks) + + def forward(self, x): + x = x.permute(0, 3, 1, 2).contiguous() + x = self.in_conv(x) + activations = {} + activations[x.shape[2]] = x + for block in self.enc_blocks: + x = block(x) + res = x.shape[2] + x = x if x.shape[1] == self.widths[res] else pad_channels(x, self.widths[res]) + activations[res] = x + return activations + + +class DecBlock(nn.Module): + def __init__(self, H, res, mixin, n_blocks): + super().__init__() + self.base = res + self.mixin = mixin + self.H = H + self.widths = get_width_settings(H.width, H.custom_width_str) + width = self.widths[res] + use_3x3 = res > 2 + cond_width = int(width * H.bottleneck_multiple) + self.zdim = H.zdim + self.enc = Block(width * 2, cond_width, H.zdim * 2, residual=False, use_3x3=use_3x3) + self.prior = Block(width, cond_width, H.zdim * 2 + width, residual=False, use_3x3=use_3x3, zero_last=True) + self.z_proj = get_1x1(H.zdim, width) + self.z_proj.weight.data *= np.sqrt(1 / n_blocks) + self.resnet = Block(width, cond_width, width, residual=True, use_3x3=use_3x3) + self.resnet.c4.weight.data *= np.sqrt(1 / n_blocks) + self.z_fn = lambda x: self.z_proj(x) + + def sample(self, x, acts): + qm, qv = self.enc(torch.cat([x, acts], dim=1)).chunk(2, dim=1) + feats = self.prior(x) + pm, pv, xpp = feats[:, :self.zdim, ...], feats[:, self.zdim:self.zdim * 2, ...], feats[:, self.zdim * 2:, ...] + x = x + xpp + z = draw_gaussian_diag_samples(qm, qv) + kl = gaussian_analytical_kl(qm, pm, qv, pv) + return z, x, kl + + def sample_uncond(self, x, t=None, lvs=None): + n, c, h, w = x.shape + feats = self.prior(x) + pm, pv, xpp = feats[:, :self.zdim, ...], feats[:, self.zdim:self.zdim * 2, ...], feats[:, self.zdim * 2:, ...] + x = x + xpp + if lvs is not None: + z = lvs + else: + if t is not None: + pv = pv + torch.ones_like(pv) * np.log(t) + z = draw_gaussian_diag_samples(pm, pv) + return z, x + + def get_inputs(self, xs, activations): + acts = activations[self.base] + try: + x = xs[self.base] + except KeyError: + x = torch.zeros_like(acts) + if acts.shape[0] != x.shape[0]: + x = x.repeat(acts.shape[0], 1, 1, 1) + return x, acts + + def forward(self, xs, activations, get_latents=False): + x, acts = self.get_inputs(xs, activations) + if self.mixin is not None: + x = x + F.interpolate(xs[self.mixin][:, :x.shape[1], ...], scale_factor=self.base // self.mixin) + z, x, kl = self.sample(x, acts) + x = x + self.z_fn(z) + x = self.resnet(x) + xs[self.base] = x + if get_latents: + return xs, dict(z=z.detach(), kl=kl) + return xs, dict(kl=kl) + + def forward_uncond(self, xs, t=None, lvs=None): + try: + x = xs[self.base] + except KeyError: + ref = xs[list(xs.keys())[0]] + x = torch.zeros(dtype=ref.dtype, size=(ref.shape[0], self.widths[self.base], self.base, self.base), device=ref.device) + if self.mixin is not None: + x = x + F.interpolate(xs[self.mixin][:, :x.shape[1], ...], scale_factor=self.base // self.mixin) + z, x = self.sample_uncond(x, t, lvs=lvs) + x = x + self.z_fn(z) + x = self.resnet(x) + xs[self.base] = x + return xs + + +class Decoder(HModule): + + def build(self): + H = self.H + resos = set() + dec_blocks = [] + self.widths = get_width_settings(H.width, H.custom_width_str) + blocks = parse_layer_string(H.dec_blocks) + for idx, (res, mixin) in enumerate(blocks): + dec_blocks.append(DecBlock(H, res, mixin, n_blocks=len(blocks))) + resos.add(res) + self.resolutions = sorted(resos) + self.dec_blocks = nn.ModuleList(dec_blocks) + self.bias_xs = nn.ParameterList([nn.Parameter(torch.zeros(1, self.widths[res], res, res)) for res in self.resolutions if res <= H.no_bias_above]) + self.out_net = DmolNet(H) + self.gain = nn.Parameter(torch.ones(1, H.width, 1, 1)) + self.bias = nn.Parameter(torch.zeros(1, H.width, 1, 1)) + self.final_fn = lambda x: x * self.gain + self.bias + + def forward(self, activations, get_latents=False): + stats = [] + xs = {a.shape[2]: a for a in self.bias_xs} + for block in self.dec_blocks: + xs, block_stats = block(xs, activations, get_latents=get_latents) + stats.append(block_stats) + xs[self.H.image_size] = self.final_fn(xs[self.H.image_size]) + return xs[self.H.image_size], stats + + def forward_uncond(self, n, t=None, y=None): + xs = {} + for bias in self.bias_xs: + xs[bias.shape[2]] = bias.repeat(n, 1, 1, 1) + for idx, block in enumerate(self.dec_blocks): + try: + temp = t[idx] + except TypeError: + temp = t + xs = block.forward_uncond(xs, temp) + xs[self.H.image_size] = self.final_fn(xs[self.H.image_size]) + return xs[self.H.image_size] + + def forward_manual_latents(self, n, latents, t=None): + xs = {} + for bias in self.bias_xs: + xs[bias.shape[2]] = bias.repeat(n, 1, 1, 1) + for block, lvs in itertools.zip_longest(self.dec_blocks, latents): + xs = block.forward_uncond(xs, t, lvs=lvs) + xs[self.H.image_size] = self.final_fn(xs[self.H.image_size]) + return xs[self.H.image_size] + + +class VAE(HModule): + def build(self): + self.encoder = Encoder(self.H) + self.decoder = Decoder(self.H) + + def forward(self, x, x_target): + activations = self.encoder.forward(x) + px_z, stats = self.decoder.forward(activations) + distortion_per_pixel = self.decoder.out_net.nll(px_z, x_target) + rate_per_pixel = torch.zeros_like(distortion_per_pixel) + ndims = np.prod(x.shape[1:]) + for statdict in stats: + rate_per_pixel += statdict['kl'].sum(dim=(1, 2, 3)) + rate_per_pixel /= ndims + elbo = (distortion_per_pixel + rate_per_pixel).mean() + return dict(elbo=elbo, distortion=distortion_per_pixel.mean(), rate=rate_per_pixel.mean()) + + def forward_get_latents(self, x): + activations = self.encoder.forward(x) + _, stats = self.decoder.forward(activations, get_latents=True) + return stats + + def forward_uncond_samples(self, n_batch, t=None): + px_z = self.decoder.forward_uncond(n_batch, t=t) + return self.decoder.out_net.sample(px_z) + + def forward_samples_set_latents(self, n_batch, latents, t=None): + px_z = self.decoder.forward_manual_latents(n_batch, latents, t=t) + return self.decoder.out_net.sample(px_z) diff --git a/vdvae/vae_helpers.py b/vdvae/vae_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..60f6619e30a48243e4a6796d9847f806eb7fbcf3 --- /dev/null +++ b/vdvae/vae_helpers.py @@ -0,0 +1,165 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +@torch.jit.script +def gaussian_analytical_kl(mu1, mu2, logsigma1, logsigma2): + return -0.5 + logsigma2 - logsigma1 + 0.5 * (logsigma1.exp() ** 2 + (mu1 - mu2) ** 2) / (logsigma2.exp() ** 2) + + +@torch.jit.script +def draw_gaussian_diag_samples(mu, logsigma): + eps = torch.empty_like(mu).normal_(0., 1.) + return torch.exp(logsigma) * eps + mu + + +def get_conv(in_dim, out_dim, kernel_size, stride, padding, zero_bias=True, zero_weights=False, groups=1, scaled=False): + c = nn.Conv2d(in_dim, out_dim, kernel_size, stride, padding, groups=groups) + if zero_bias: + c.bias.data *= 0.0 + if zero_weights: + c.weight.data *= 0.0 + return c + + +def get_3x3(in_dim, out_dim, zero_bias=True, zero_weights=False, groups=1, scaled=False): + return get_conv(in_dim, out_dim, 3, 1, 1, zero_bias, zero_weights, groups=groups, scaled=scaled) + + +def get_1x1(in_dim, out_dim, zero_bias=True, zero_weights=False, groups=1, scaled=False): + return get_conv(in_dim, out_dim, 1, 1, 0, zero_bias, zero_weights, groups=groups, scaled=scaled) + + +def log_prob_from_logits(x): + """ numerically stable log_softmax implementation that prevents overflow """ + axis = len(x.shape) - 1 + m = x.max(dim=axis, keepdim=True)[0] + return x - m - torch.log(torch.exp(x - m).sum(dim=axis, keepdim=True)) + + +def const_max(t, constant): + other = torch.ones_like(t) * constant + return torch.max(t, other) + + +def const_min(t, constant): + other = torch.ones_like(t) * constant + return torch.min(t, other) + + +def discretized_mix_logistic_loss(x, l, low_bit=False): + """ log-likelihood for mixture of discretized logistics, assumes the data has been rescaled to [-1,1] interval """ + # Adapted from https://github.com/openai/pixel-cnn/blob/master/pixel_cnn_pp/nn.py + xs = [s for s in x.shape] # true image (i.e. labels) to regress to, e.g. (B,32,32,3) + ls = [s for s in l.shape] # predicted distribution, e.g. (B,32,32,100) + nr_mix = int(ls[-1] / 10) # here and below: unpacking the params of the mixture of logistics + logit_probs = l[:, :, :, :nr_mix] + l = torch.reshape(l[:, :, :, nr_mix:], xs + [nr_mix * 3]) + means = l[:, :, :, :, :nr_mix] + log_scales = const_max(l[:, :, :, :, nr_mix:2 * nr_mix], -7.) + coeffs = torch.tanh(l[:, :, :, :, 2 * nr_mix:3 * nr_mix]) + x = torch.reshape(x, xs + [1]) + torch.zeros(xs + [nr_mix]).to(x.device) # here and below: getting the means and adjusting them based on preceding sub-pixels + m2 = torch.reshape(means[:, :, :, 1, :] + coeffs[:, :, :, 0, :] * x[:, :, :, 0, :], [xs[0], xs[1], xs[2], 1, nr_mix]) + m3 = torch.reshape(means[:, :, :, 2, :] + coeffs[:, :, :, 1, :] * x[:, :, :, 0, :] + coeffs[:, :, :, 2, :] * x[:, :, :, 1, :], [xs[0], xs[1], xs[2], 1, nr_mix]) + means = torch.cat([torch.reshape(means[:, :, :, 0, :], [xs[0], xs[1], xs[2], 1, nr_mix]), m2, m3], dim=3) + centered_x = x - means + inv_stdv = torch.exp(-log_scales) + if low_bit: + plus_in = inv_stdv * (centered_x + 1. / 31.) + cdf_plus = torch.sigmoid(plus_in) + min_in = inv_stdv * (centered_x - 1. / 31.) + else: + plus_in = inv_stdv * (centered_x + 1. / 255.) + cdf_plus = torch.sigmoid(plus_in) + min_in = inv_stdv * (centered_x - 1. / 255.) + cdf_min = torch.sigmoid(min_in) + log_cdf_plus = plus_in - F.softplus(plus_in) # log probability for edge case of 0 (before scaling) + log_one_minus_cdf_min = -F.softplus(min_in) # log probability for edge case of 255 (before scaling) + cdf_delta = cdf_plus - cdf_min # probability for all other cases + mid_in = inv_stdv * centered_x + log_pdf_mid = mid_in - log_scales - 2. * F.softplus(mid_in) # log probability in the center of the bin, to be used in extreme cases (not actually used in our code) + + # now select the right output: left edge case, right edge case, normal case, extremely low prob case (doesn't actually happen for us) + + # this is what we are really doing, but using the robust version below for extreme cases in other applications and to avoid NaN issue with tf.select() + # log_probs = tf.select(x < -0.999, log_cdf_plus, tf.select(x > 0.999, log_one_minus_cdf_min, tf.log(cdf_delta))) + + # robust version, that still works if probabilities are below 1e-5 (which never happens in our code) + # tensorflow backpropagates through tf.select() by multiplying with zero instead of selecting: this requires use to use some ugly tricks to avoid potential NaNs + # the 1e-12 in tf.maximum(cdf_delta, 1e-12) is never actually used as output, it's purely there to get around the tf.select() gradient issue + # if the probability on a sub-pixel is below 1e-5, we use an approximation based on the assumption that the log-density is constant in the bin of the observed sub-pixel value + if low_bit: + log_probs = torch.where(x < -0.999, + log_cdf_plus, + torch.where(x > 0.999, + log_one_minus_cdf_min, + torch.where(cdf_delta > 1e-5, + torch.log(const_max(cdf_delta, 1e-12)), + log_pdf_mid - np.log(15.5)))) + else: + log_probs = torch.where(x < -0.999, + log_cdf_plus, + torch.where(x > 0.999, + log_one_minus_cdf_min, + torch.where(cdf_delta > 1e-5, + torch.log(const_max(cdf_delta, 1e-12)), + log_pdf_mid - np.log(127.5)))) + log_probs = log_probs.sum(dim=3) + log_prob_from_logits(logit_probs) + mixture_probs = torch.logsumexp(log_probs, -1) + return -1. * mixture_probs.sum(dim=[1, 2]) / np.prod(xs[1:]) + + +def sample_from_discretized_mix_logistic(l, nr_mix): + ls = [s for s in l.shape] + xs = ls[:-1] + [3] + # unpack parameters + logit_probs = l[:, :, :, :nr_mix] + l = torch.reshape(l[:, :, :, nr_mix:], xs + [nr_mix * 3]) + # sample mixture indicator from softmax + eps = torch.empty(logit_probs.shape, device=l.device).uniform_(1e-5, 1. - 1e-5) + amax = torch.argmax(logit_probs - torch.log(-torch.log(eps)), dim=3) + sel = F.one_hot(amax, num_classes=nr_mix).float() + sel = torch.reshape(sel, xs[:-1] + [1, nr_mix]) + # select logistic parameters + means = (l[:, :, :, :, :nr_mix] * sel).sum(dim=4) + log_scales = const_max((l[:, :, :, :, nr_mix:nr_mix * 2] * sel).sum(dim=4), -7.) + coeffs = (torch.tanh(l[:, :, :, :, nr_mix * 2:nr_mix * 3]) * sel).sum(dim=4) + # sample from logistic & clip to interval + # we don't actually round to the nearest 8bit value when sampling + u = torch.empty(means.shape, device=means.device).uniform_(1e-5, 1. - 1e-5) + x = means + torch.exp(log_scales) * (torch.log(u) - torch.log(1. - u)) + x0 = const_min(const_max(x[:, :, :, 0], -1.), 1.) + x1 = const_min(const_max(x[:, :, :, 1] + coeffs[:, :, :, 0] * x0, -1.), 1.) + x2 = const_min(const_max(x[:, :, :, 2] + coeffs[:, :, :, 1] * x0 + coeffs[:, :, :, 2] * x1, -1.), 1.) + return torch.cat([torch.reshape(x0, xs[:-1] + [1]), torch.reshape(x1, xs[:-1] + [1]), torch.reshape(x2, xs[:-1] + [1])], dim=3) + + +class HModule(nn.Module): + def __init__(self, H): + super().__init__() + self.H = H + self.build() + + +class DmolNet(nn.Module): + def __init__(self, H): + super().__init__() + self.H = H + self.width = H.width + self.out_conv = get_conv(H.width, H.num_mixtures * 10, kernel_size=1, stride=1, padding=0) + + def nll(self, px_z, x): + return discretized_mix_logistic_loss(x=x, l=self.forward(px_z), low_bit=self.H.dataset in ['ffhq_256']) + + def forward(self, px_z): + xhat = self.out_conv(px_z) + return xhat.permute(0, 2, 3, 1) + + def sample(self, px_z): + im = sample_from_discretized_mix_logistic(self.forward(px_z), self.H.num_mixtures) + xhat = (im + 1.0) * 127.5 + xhat = xhat.detach().cpu().numpy() + xhat = np.minimum(np.maximum(0.0, xhat), 255.0).astype(np.uint8) + return xhat diff --git a/versatile_diffusion/LICENSE b/versatile_diffusion/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..8ea0dead0eae2bb8274320bfb16f633c3f386dda --- /dev/null +++ b/versatile_diffusion/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 SHI Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/versatile_diffusion/README.md b/versatile_diffusion/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b4de1aba25bc0713f123da94a6d8edaff5bd8833 --- /dev/null +++ b/versatile_diffusion/README.md @@ -0,0 +1,175 @@ +# Versatile Diffusion + +[![Huggingface space](https://img.shields.io/badge/🤗-Huggingface%20Space-cyan.svg)](https://huggingface.co/spaces/shi-labs/Versatile-Diffusion) +[![Framework: PyTorch](https://img.shields.io/badge/Framework-PyTorch-orange.svg)](https://pytorch.org/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +This repo hosts the official implementary of: + +[Xingqian Xu](https://ifp-uiuc.github.io/), Atlas Wang, Eric Zhang, Kai Wang, and [Humphrey Shi](https://www.humphreyshi.com/home), **Versatile Diffusion: Text, Images and Variations All in One Diffusion Model**, [Paper arXiv Link](https://arxiv.org/abs/2211.08332). + +## News + +- [2022.11.23]: Versatile Diffusion are now released in [```diffuser==0.8.0```](https://github.com/huggingface/diffusers/releases) library🚀! +- [2022.11.21]: ```inference.py``` now supports ```--fp16```, reducing both GPU memory and model size by half. +- [2022.11.19]: We add a simplified evaluation script ```inference.py``` that supports all mentioned applications. +- **[2022.11.16]: Our demo is up and running on [🤗HuggingFace](https://huggingface.co/spaces/shi-labs/Versatile-Diffusion)!** +- [2022.11.14]: Part of our evaluation code and models are released! +- [2022.11.12]: Repo initiated + +## Introduction + +We built **Versatile Diffusion (VD), the first unified multi-flow multimodal diffusion framework**, as a step towards **Universal Generative AI**. Versatile Diffusion can natively support image-to-text, image-variation, text-to-image, and text-variation, and can be further extended to other applications such as semantic-style disentanglement, image-text dual-guided generation, latent image-to-text-to-image editing, and more. Future versions will support more modalities such as speech, music, video and 3D. + +

+ +

+ +## Network and Framework + +One single flow of Versatile Diffusion contains a VAE, a diffuser, and a context encoder, and thus handles one task (e.g., text-to-image) under one data type (e.g., image) and one context type (e.g., text). The multi-flow structure of Versatile Diffusion shows in the following diagram: + +

+ +

+ +According to Versatile Diffusion, we further proposed a generalized multi-flow multimodal framework with VAEs, context encoders, and diffusers containing three layers (i.e., global, data, and context layers). To involve a new multimodal task in this framework, we bring out the following requirements: + +* The design of the core diffuser should contain shared global layers, swappable data, and context layers that will be correspondingly activated based on data and context types. +* The choice of VAEs should smoothly map data onto highly interpretable latent spaces. +* The choice of context encoders should jointly minimize the cross-modal statistical distance on all supported content types. + + +## Performance + +

+ + + +

+ +## Data + +We use Laion2B-en with customized data filters as our main dataset. Since Laion2B is very large and typical training is less than one epoch, we usually do not need to download the complete dataset for training. Same story for VDs. + +Directory of Laion2B for our code: + +``` +├── data +│   └── laion2b +│   └── data +│    └── 00000.tar +│    └── 00000.parquet +│    └── 00000_stats.jsom_ +│    └── 00001.tar +│    └── ... +``` + +These compressed data are generated with img2dataset API [official github link](https://github.com/rom1504/img2dataset). + +## Setup + +``` +conda create -n versatile-diffusion python=3.8 +conda activate versatile-diffusion +conda install pytorch==1.12.1 torchvision=0.13.1 -c pytorch +pip install -r requirement.txt +``` + +## Pretrained models + +Useful pretrained models can be downloaded from HuggingFace [link](https://huggingface.co/shi-labs/versatile-diffusion/tree/main/pretrained_pth). The pretrained folder should include the following files: + +``` +├── pretrained +│   └── kl-f8.pth +│   └── optimus-vae.pth +│ └── vd-four-flow-v1-0.pth (originally named vd-official.pth) +│ └── vd-four-flow-v1-0-fp16.pth +``` + +Model named with **-fp16** are models with float16 parameters and correpondingly used in ```--fp16``` evaluation. The float16 models are half of the size comparing to the float32 models. + +Please refer to this [doc](https://github.com/SHI-Labs/Versatile-Diffusion/blob/master/README_extra.md) to download VD-DC and baselines models. + +## Evaluation + +Here is the one-line shell command to evaluate VD four-flow models with multiple GPUs. + +``` +python main.py --config vd_official_eval --gpu 0 1 2 3 4 5 6 7 --eval 99999 +``` + +The corresponding evaluation configs can be found in ```./configs/experiment/vd_official_eval.yaml```. There are useful information in the config. You can easy customized it and run your own batched evaluations. + +For the commands above, you also need to: +* Create ```./pretrained``` and move all downloaded pretrained models in it. +* Create ```./log/vd_nodataset/99999_eval``` for evaluations on Versatile Diffusion + +Besides, we provide a simple evaluation script ```inference.py``` that support all applications mentioned in the paper. Examples are below: + +``` +python inference.py --gpu 0 --app text-to-image --prompt "a dream of a village in china, by Caspar David Friedrich, matte painting trending on artstation HQ" --seed 0 --save log/text-to-image.png +python inference.py --gpu 0 --app image-variation --image assets/space.jpg --seed 8 --save log/image-variation.png --coloradj simple +python inference.py --gpu 0 --app image-to-text --image assets/space.jpg --seed 8 +python inference.py --gpu 0 --app text-variation --prompt "a pink car" --seed 8 +python inference.py --gpu 0 --app disentanglement --image assets/vermeer.jpg --seed 8 --save log/disentanglement.png --coloradj simple --dislevel -2 +python inference.py --gpu 0 --app dual-guided --image assets/benz.jpg --prompt "cyberpunk 2077" --seed 22 --save log/dual-guided.png --coloradj none --dgmixing 0.7 +python inference.py --gpu 0 --app i2t2i --image assets/ghibli.jpg --nprompt "white house" --pprompt "tall castle" --seed 20 --save log/i2t2i.png --coloradj simple +``` + +You can also inference with float16 by adding a tag ```--fp16``` to each command. It doubles the running speed using a half size pretrained model ```vd-***-fp16.pth```. You will also need to update transformers to 4.24.0. Below is an example command using float16: + +``` +python inference.py --gpu 0 --app image-variation --image assets/space.jpg --seed 8 --save log/image-variation.png --coloradj simple --fp16 +``` + +For baseline experiments and experiments on VD-DC, please refer to this [doc](https://github.com/SHI-Labs/Versatile-Diffusion/blob/master/README_extra.md) + +## Training + +Coming soon + +## Gallery + +Text-to-Image + +

+ +

+ +Image-Variation + +

+ +

+ +Image-Variation with Semantic Focus + +

+ +

+ +Dual-guided + +

+ +

+ +## Citation + +``` +@article{xu2022versatile, + title = {Versatile Diffusion: Text, Images and Variations All in One Diffusion Model}, + author = {Xingqian Xu, Zhangyang Wang, Eric Zhang, Kai Wang, Humphrey Shi}, + year = 2022, + url = {https://arxiv.org/abs/2211.08332}, + eprint = {2211.08332}, + archiveprefix = {arXiv}, + primaryclass = {cs.CV} +} +``` + +## Acknowledgement + +Part of the codes reorganizes/reimplements code from the following repositories: [LDM official Github](https://github.com/CompVis/latent-diffusion), which also oriented from [DDPM official Github](https://github.com/lucidrains/denoising-diffusion-pytorch). diff --git a/versatile_diffusion/README_extra.md b/versatile_diffusion/README_extra.md new file mode 100644 index 0000000000000000000000000000000000000000..49b67f0e557a0fd9a0a77db59b31a9b5c011b9af --- /dev/null +++ b/versatile_diffusion/README_extra.md @@ -0,0 +1,23 @@ +## Baselines and VD-DC + +Some non-core models can be downloaded from this [link](https://drive.google.com/drive/folders/1SloRnOO9UnonfvubPWfw0uFpLco_2JvH?usp=sharing). +It contains two baseline models: ```sd-v1-4.pth``` and ```sd-variation.pth```, and our VD-DC model ```vd-dc.pth``` + +All models should be copyed to ```pretrained``` folder. + +To evaluate baseline experiments: + +``` +python main.py --config sd_eval --gpu 0 1 2 3 4 5 6 7 --eval 99999 +python main.py --config sd_variation_eval --gpu 0 1 2 3 4 5 6 7 --eval 99999 +``` + +You will need to create ```./log/sd_nodataset/99999_eval``` to make these baseline evaluations running. + +To evaluate VD-DC experiments: + +``` +python main.py --config vd_dc_eval --gpu 0 1 2 3 4 5 6 7 --eval 99999 +``` + +Similarly, you will need to create ```./log/vd_nodataset/99999_eval``` to make the evaluation running. diff --git a/versatile_diffusion/configs/dataset/laion2b.yaml b/versatile_diffusion/configs/dataset/laion2b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ba06c7f77951d8b6e7dea5a3f6f8070d88474ff --- /dev/null +++ b/versatile_diffusion/configs/dataset/laion2b.yaml @@ -0,0 +1,46 @@ +laion2b: + symbol: laion2b + type: laion2b + root_dir: data/laion2b/ + pin_memory: true + +laion2b_dummy: + super_cfg: laion2b + type: laion2b_dummy + +############################### +# webdataset based dataloader # +############################### + +laion2b_webdataset: + super_cfg: laion2b + type: laion2b_webdataset + +laion2b_webdataset_256: + super_cfg: laion2b_webdataset + scale: 256 + min_size: 224 + +laion2b_webdataset_256_debug: + super_cfg: laion2b_webdataset_256 + root_dir: data/laion2b-debug/ + +laion2b_webdataset_512: + super_cfg: laion2b_webdataset + scale: 512 + min_size: 448 + +laion2b_webdataset_512_debug: + super_cfg: laion2b_webdataset_512 + root_dir: data/laion2b-debug/ + +laion2b_webdataset_512_sdofficial: + super_cfg: laion2b + type: laion2b_webdataset_sdofficial + scale: 512 + min_size: 448 + +laion2b_webdataset_512_sdofficial_debug: + super_cfg: laion2b_webdataset_512_sdofficial + root_dir: data/laion2b-debug/ + shuffle: 0 diff --git a/versatile_diffusion/configs/experiment/sd_eval.yaml b/versatile_diffusion/configs/experiment/sd_eval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3ca62ed3ec91acf82c064d0c04d8753b9b08040 --- /dev/null +++ b/versatile_diffusion/configs/experiment/sd_eval.yaml @@ -0,0 +1,47 @@ +env: + debug: false + cuda: true + dist_backend: nccl + matplotlib_mode: agg + log_root_dir: log + rnd_seed: 20 + +model: MODEL(sd_t2i_fullclip_backward_compatible) + +eval: + main: lib.experiments.sd_default.eval + stage: lib.experiments.sd_default.eval_stage + dataset: null + + conditioning: + - a dream of a village in china, by Caspar David Friedrich, matte painting trending on artstation HQ + - a beautiful grand nebula in the universe + - area of rocks that deep inside the forest, divine domain + - heavy arms gundam penguin mech + - realistic scenery of houston texas city view under a starry sky in hyperrealistic style and ultra HD, 8k + - red maple on a hill in golden autumn + - man standing on the beach near sea + - blue and yellow balloons in the sky + + replicate: 1 + + sample: + output_dim: [512, 512] + n_samples: 4 + ddim_steps: 50 + ddim_eta: 0.0 + scale: 5.0 + + batch_size_per_gpu: 0 + batch_size: null + dataset_num_workers_per_gpu: 0 + dataset_num_workers: null + + evaluator: null + log_every: null + + pretrained_pth: pretrained/sd-v1-4.pth + strict_sd: true + + fix_seed: true + eval_subdir: sd_v1_4 diff --git a/versatile_diffusion/configs/experiment/sd_variation_eval.yaml b/versatile_diffusion/configs/experiment/sd_variation_eval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..794e093d7619a6a720b0cfc08746c744eed1d37f --- /dev/null +++ b/versatile_diffusion/configs/experiment/sd_variation_eval.yaml @@ -0,0 +1,57 @@ +env: + debug: false + cuda: true + dist_backend: nccl + matplotlib_mode: agg + log_root_dir: log + rnd_seed: 200 + +model: MODEL(sd_variation) +eval: + main: lib.experiments.sd_default.eval + stage: lib.experiments.sd_default.eval_stage_variation + dataset: null + save_code: true + + conditioning: + - assets/benz.jpg + - assets/ghibli.jpg + - assets/horse.png + - assets/matisse.jpg + - assets/penguin.png + - assets/scream.jpg + - assets/space.jpg + - assets/vermeer.jpg + - assets/boy_and_girl.jpg + - assets/church.jpg + - assets/firework.jpg + - assets/house_by_lake.jpg + - assets/night_light.jpg + - assets/san_diego.jpg + - assets/tiger.jpg + - assets/train.jpg + + replicate: 1 + + sample: + output_dim: [512, 512] + n_samples: 4 + ddim_steps: 50 + ddim_eta: 0.0 + scale: 7.5 + + color_adj: true + color_adj_keep_ratio: 0.5 + color_adj_simple: true + + batch_size_per_gpu: 0 + batch_size: null + dataset_num_workers_per_gpu: 0 + dataset_num_workers: null + + pretrained_pth_ema: pretrained/sd-variation-ema.pth + strict_sd: true + + is_lite: true + fix_seed: true + eval_subdir: sd_variation diff --git a/versatile_diffusion/configs/experiment/vd_dc_eval.yaml b/versatile_diffusion/configs/experiment/vd_dc_eval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f40f09fb18a0a42db95d85666b60b7084206933 --- /dev/null +++ b/versatile_diffusion/configs/experiment/vd_dc_eval.yaml @@ -0,0 +1,57 @@ +env: + debug: false + cuda: true + dist_backend: nccl + matplotlib_mode: agg + log_root_dir: log + rnd_seed: 200 + +model: MODEL(vd_dc_noema) + +eval: + main: lib.experiments.vd_default.eval + stage: lib.experiments.vd_default.eval_stage_dc + dataset: null + save_code: true + + conditioning: + - a dream of a village in china, by Caspar David Friedrich, matte painting trending on artstation HQ + # - a beautiful grand nebula in the universe + # - area of rocks that deep inside the forest, divine domain + # - heavy arms gundam penguin mech + # - realistic scenery of houston texas city view under a starry sky in hyperrealistic style and ultra HD, 8k + # - red maple on a hill in golden autumn + # - man standing on the beach near sea + # - blue and yellow balloons in the sky + - assets/benz.jpg + # - assets/ghibli.jpg + # - assets/horse.png + # - assets/matisse.jpg + # - assets/penguin.png + # - assets/scream.jpg + # - assets/space.jpg + # - assets/vermeer.jpg + + replicate: 1 + + sample: + output_dim: [512, 512] + n_samples: 4 + ddim_steps: 50 + ddim_eta: 0.0 + scale: 7.5 + + color_adj: true + color_adj_keep_ratio: 0.5 + color_adj_simple: true + + batch_size_per_gpu: 0 + batch_size: null + dataset_num_workers_per_gpu: 0 + dataset_num_workers: null + + pretrained_pth: pretrained/vd-dc.pth + strict_sd: true + + fix_seed: true + eval_subdir: vd_dc diff --git a/versatile_diffusion/configs/experiment/vd_official_eval.yaml b/versatile_diffusion/configs/experiment/vd_official_eval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c278ec965237d7d17a5f7b1290aef32316563c00 --- /dev/null +++ b/versatile_diffusion/configs/experiment/vd_official_eval.yaml @@ -0,0 +1,63 @@ +env: + debug: false + cuda: true + dist_backend: nccl + matplotlib_mode: agg + log_root_dir: log + rnd_seed: 20 + +model: MODEL(vd_noema) + +eval: + main: lib.experiments.vd_default.eval + stage: lib.experiments.vd_default.eval_stage + + dataset: null + + conditioning: + - ["a dream of a village in china, by Caspar David Friedrich, matte painting trending on artstation HQ", "image"] + - ["a beautiful grand nebula in the universe", "image"] + - ["area of rocks that deep inside the forest, divine domain", "image"] + - ["heavy arms gundam penguin mech", "image"] + - ["assets/boy_and_girl.jpg", "image"] + - ["assets/house_by_lake.jpg", "image"] + - ["assets/ghibli.jpg", "image"] + - ["assets/scream.jpg", "image"] + + - ["a dream of a village in china, by Caspar David Friedrich, matte painting trending on artstation HQ", "text"] + - ["a beautiful grand nebula in the universe", "text"] + - ["area of rocks that deep inside the forest, divine domain", "text"] + - ["heavy arms gundam penguin mech", "text"] + - ["assets/boy_and_girl.jpg", "text"] + - ["assets/house_by_lake.jpg", "text"] + - ["assets/ghibli.jpg", "text"] + - ["assets/scream.jpg", "text"] + + replicate: 1 + + sample: + image_output_dim: [512, 512] + text_latent_dim: 768 + n_samples: 4 + ddim_steps: 50 + ddim_eta: 0.0 + scale: 7.5 + + # Some useful post processing + prompt_temperature: 1.0 + prompt_merge_same_adj_word: true + color_adj: true + color_adj_keep_ratio: 0.5 + color_adj_simple: true + + batch_size_per_gpu: 0 + batch_size: null + dataset_num_workers_per_gpu: 0 + dataset_num_workers: null + + pretrained_pth: pretrained/vd-four-flow-v1-0.pth + strict_sd: true + + is_lite: true + fix_seed: true + eval_subdir: vd-four-flow-v1-0-results diff --git a/versatile_diffusion/configs/model/clip.yaml b/versatile_diffusion/configs/model/clip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..349c5c596e645c291c0edbcf253bc6b7ea15126c --- /dev/null +++ b/versatile_diffusion/configs/model/clip.yaml @@ -0,0 +1,50 @@ + +clip: + symbol: clip + args: {} + +clip_frozen: + super_cfg: clip + type: clip_frozen + args: {} + +clip_text_frozen: + super_cfg: clip + type: clip_text_frozen + args: {} + +clip_vision_frozen: + super_cfg: clip + type: clip_vision_frozen + args: {} + +############################ +# clip with focused encode # +############################ + +clip_frozen_encode_text: + super_cfg: clip + type: clip_frozen + args: + encode_type : encode_text + +clip_frozen_encode_vision: + super_cfg: clip + type: clip_frozen + args: + encode_type : encode_vision + +clip_frozen_encode_text_noproj: + super_cfg: clip + type: clip_frozen + args: + encode_type : encode_text_noproj + +##################################### +# clip vision forzen justin version # +##################################### + +clip_vision_frozen_justin: + super_cfg: clip + type: clip_vision_frozen_justin + args: {} diff --git a/versatile_diffusion/configs/model/openai_unet.yaml b/versatile_diffusion/configs/model/openai_unet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bf09d5af5e9776a6a1314915e00114dbac4450d --- /dev/null +++ b/versatile_diffusion/configs/model/openai_unet.yaml @@ -0,0 +1,72 @@ +openai_unet_sd: + type: openai_unet + args: + image_size: null # no use + in_channels: 4 + out_channels: 4 + model_channels: 320 + attention_resolutions: [ 4, 2, 1 ] + num_res_blocks: [ 2, 2, 2, 2 ] + channel_mult: [ 1, 2, 4, 4 ] + # disable_self_attentions: [ False, False, False, False ] # converts the self-attention to a cross-attention layer if true + num_heads: 8 + use_spatial_transformer: True + transformer_depth: 1 + context_dim: 768 + use_checkpoint: True + legacy: False + +openai_unet_dual_context: + super_cfg: openai_unet_sd + type: openai_unet_dual_context + +######################## +# Code cleaned version # +######################## + +openai_unet_2d: + type: openai_unet_2d + args: + input_channels: 4 + model_channels: 320 + output_channels: 4 + num_noattn_blocks: [ 2, 2, 2, 2 ] + channel_mult: [ 1, 2, 4, 4 ] + with_attn: [true, true, true, false] + num_heads: 8 + context_dim: 768 + use_checkpoint: True + +openai_unet_0d: + type: openai_unet_0d + args: + input_channels: 768 + model_channels: 320 + output_channels: 768 + num_noattn_blocks: [ 2, 2, 2, 2 ] + channel_mult: [ 1, 2, 4, 4 ] + with_attn: [true, true, true, false] + num_heads: 8 + context_dim: 768 + use_checkpoint: True + +openai_unet_0dmd: + type: openai_unet_0dmd + args: + input_channels: 768 + model_channels: 320 + output_channels: 768 + num_noattn_blocks: [ 2, 2, 2, 2 ] + channel_mult: [ 1, 2, 4, 4 ] + second_dim: [ 4, 4, 4, 4 ] + with_attn: [true, true, true, false] + num_heads: 8 + context_dim: 768 + use_checkpoint: True + +openai_unet_vd: + type: openai_unet_vd + args: + unet_image_cfg: MODEL(openai_unet_2d) + unet_text_cfg: MODEL(openai_unet_0dmd) + \ No newline at end of file diff --git a/versatile_diffusion/configs/model/optimus.yaml b/versatile_diffusion/configs/model/optimus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45d2aedcb9ee674d792cd723cca6962db2cd0e68 --- /dev/null +++ b/versatile_diffusion/configs/model/optimus.yaml @@ -0,0 +1,102 @@ + +optimus: + symbol: optimus + find_unused_parameters: false + args: {} + +optimus_bert_encoder: + super_cfg: optimus + type: optimus_bert_connector + # pth: pretrained/optimus_bert_encoder.pth + args: + config: + architectures: + - BertForMaskedLM + attention_probs_dropout_prob: 0.1 + finetuning_task: null + hidden_act: gelu + hidden_dropout_prob: 0.1 + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + layer_norm_eps: 1.e-12 + max_position_embeddings: 512 + num_attention_heads: 12 + num_hidden_layers: 12 + num_labels: 2 + output_attentions: false + output_hidden_states: false + pruned_heads: {} + torchscript: false + type_vocab_size: 2 + vocab_size: 28996 + latent_size: 768 + +optimus_bert_tokenizer: + super_cfg: optimus + type: optimus_bert_tokenizer + args: + do_lower_case: false + max_len: 512 + vocab_file: versatile_diffusion/lib/model_zoo/optimus_models/vocab/bert-base-cased-vocab.txt + +optimus_gpt2_decoder: + super_cfg: optimus + type: optimus_gpt2_connector + # pth: pretrained/optimus_gpt2_decoder.pth + args: + config: + architectures: + - GPT2LMHeadModel + attn_pdrop: 0.1 + embd_pdrop: 0.1 + finetuning_task: null + hidden_size: 768 + initializer_range: 0.02 + latent_size: 768 + layer_norm_epsilon: 1.e-05 + max_position_embeddings: 1024 + n_ctx: 1024 + n_embd: 768 + n_head: 12 + n_layer: 12 + n_positions: 1024 + num_attention_heads: 12 + num_hidden_layers: 12 + num_labels: 1 + output_attentions: false + output_hidden_states: false + pretrained_config_archive_map: + gpt2 : https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json + gpt2-medium : https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json + gpt2-large : https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json + pruned_heads: {} + resid_pdrop: 0.1 + summary_activation: null + summary_first_dropout: 0.1 + summary_proj_to_labels: true + summary_type: cls_index + summary_use_proj: true + torchscript: false + vocab_size: 50260 + +optimus_gpt2_tokenizer: + super_cfg: optimus + type: optimus_gpt2_tokenizer + args: + do_lower_case: false + max_len: 1024 + vocab_file: versatile_diffusion/lib/model_zoo/optimus_models/vocab/gpt2-vocab.json + merges_file: versatile_diffusion/lib/model_zoo/optimus_models/vocab/gpt2-merges.txt + +optimus_vae: + super_cfg: optimus + type: optimus_vae + pth: versatile_diffusion/pretrained/optimus-vae.pth + args: + encoder: MODEL(optimus_bert_encoder) + decoder: MODEL(optimus_gpt2_decoder) + tokenizer_encoder: MODEL(optimus_bert_tokenizer) + tokenizer_decoder: MODEL(optimus_gpt2_tokenizer) + args: + latent_size: 768 diff --git a/versatile_diffusion/configs/model/sd.yaml b/versatile_diffusion/configs/model/sd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caeda140d16c338a661901e77f2200694916b608 --- /dev/null +++ b/versatile_diffusion/configs/model/sd.yaml @@ -0,0 +1,68 @@ +sd_base: + symbol: sd + find_unused_parameters: true + +sd_autoencoder: + type: autoencoderkl + args: + embed_dim: 4 + monitor: val/rec_loss + ddconfig: + double_z: true + z_channels: 4 + resolution: 256 + in_channels: 3 + out_ch: 3 + ch: 128 + ch_mult: [1, 2, 4, 4] + num_res_blocks: 2 + attn_resolutions: [] + dropout: 0.0 + lossconfig: + target: torch.nn.Identity + pth: versatile_diffusion/pretrained/kl-f8.pth + +sd_t2i: + super_cfg: sd_base + type: sd_t2i + args: + first_stage_config: MODEL(sd_autoencoder) + cond_stage_config: MODEL(clip_text_frozen) + unet_config: MODEL(openai_unet_sd) + beta_linear_start: 0.00085 + beta_linear_end: 0.012 + num_timesteps_cond: 1 + timesteps: 1000 + scale_factor: 0.18215 + use_ema: true + +sd_t2i_noema: + super_cfg: sd + args: + use_ema: false + +##################### +# sd with full clip # +##################### + +sd_t2i_fullclip_backward_compatible: + super_cfg: sd_t2i + args: + cond_stage_config: MODEL(clip_frozen_encode_text_noproj) + +sd_t2i_fullclip_backward_compatible_noema: + super_cfg: sd_t2i_noema + args: + cond_stage_config: MODEL(clip_frozen_encode_text_noproj) + +sd_t2i_fullclip: + super_cfg: sd_t2i + args: + cond_stage_config: MODEL(clip_frozen_encode_text) + +sd_variation: + super_cfg: sd_t2i + type: sd_variation + args: + cond_stage_config: MODEL(clip_vision_frozen_justin) + diff --git a/versatile_diffusion/configs/model/vd.yaml b/versatile_diffusion/configs/model/vd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa9fa90382039f0d560c680cb528efea7006ec31 --- /dev/null +++ b/versatile_diffusion/configs/model/vd.yaml @@ -0,0 +1,61 @@ +# vd_base: +# symbol: vd +# find_unused_parameters: true + +############ +# vd basic # +############ + +vd_basic: + super_cfg: sd_t2i + type: vd_basic + symbol: vd + find_unused_parameters: true + args: + cond_stage_config: MODEL(clip_frozen_encode_vision) + +vd_basic_noema: + super_cfg: vd_basic + args: + use_ema: false + +################### +# vd dual-context # +################### + +vd_dc: + super_cfg: sd_t2i_fullclip + type: vd_dc + symbol: vd + find_unused_parameters: true + args: + unet_config: MODEL(openai_unet_dual_context) + +vd_dc_noema: + super_cfg: vd_dc + args: + use_ema: false + +###### +# vd # +###### + +vd: + type: vd + symbol: vd + find_unused_parameters: true + args: + autokl_cfg: MODEL(sd_autoencoder) + optimus_cfg: MODEL(optimus_vae) + clip_cfg: MODEL(clip_frozen) + unet_config: MODEL(openai_unet_vd) + beta_linear_start: 0.00085 + beta_linear_end: 0.012 + timesteps: 1000 + scale_factor: 0.18215 + use_ema: true + +vd_noema: + super_cfg: vd + args: + use_ema: false diff --git a/versatile_diffusion/inference.py b/versatile_diffusion/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f1543511428ee0326410c8f18ec5ea1222fb1c37 --- /dev/null +++ b/versatile_diffusion/inference.py @@ -0,0 +1,599 @@ +import os +import os.path as osp +import PIL +from PIL import Image +from pathlib import Path +import numpy as np +import numpy.random as npr + +import torch +import torchvision.transforms as tvtrans +from lib.cfg_helper import model_cfg_bank +from lib.model_zoo import get_model +from lib.model_zoo.ddim_dualcontext import DDIMSampler_DualContext +from lib.experiments.sd_default import color_adjust, auto_merge_imlist + +import argparse + +n_sample_image_default = 2 +n_sample_text_default = 4 + +def highlight_print(info): + print('') + print(''.join(['#']*(len(info)+4))) + print('# '+info+' #') + print(''.join(['#']*(len(info)+4))) + print('') + +class vd_inference(object): + def __init__(self, pth='pretrained/vd1.0-four-flow.pth', fp16=False, device=0): + cfgm_name = 'vd_noema' + cfgm = model_cfg_bank()('vd_noema') + device_str = device if isinstance(device, str) else 'cuda:{}'.format(device) + cfgm.args.autokl_cfg.map_location = device_str + cfgm.args.optimus_cfg.map_location = device_str + net = get_model()(cfgm) + if fp16: + highlight_print('Running in FP16') + net.clip.fp16 = True + net = net.half() + sd = torch.load(pth, map_location=device_str) + net.load_state_dict(sd, strict=False) + print('Load pretrained weight from {}'.format(pth)) + net.to(device) + + self.device = device + self.model_name = cfgm_name + self.net = net + self.fp16 = fp16 + from lib.model_zoo.ddim_vd import DDIMSampler_VD + self.sampler = DDIMSampler_VD(net) + + def regularize_image(self, x): + BICUBIC = PIL.Image.Resampling.BICUBIC + if isinstance(x, str): + x = Image.open(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, PIL.Image.Image): + x = x.resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, np.ndarray): + x = PIL.Image.fromarray(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, torch.Tensor): + pass + else: + assert False, 'Unknown image type' + + assert (x.shape[1]==512) & (x.shape[2]==512), \ + 'Wrong image size' + x = x.to(self.device) + if self.fp16: + x = x.half() + return x + + def decode(self, z, xtype, ctype, color_adj='None', color_adj_to=None): + net = self.net + if xtype == 'image': + x = net.autokl_decode(z) + + color_adj_flag = (color_adj!='none') and (color_adj!='None') and (color_adj is not None) + color_adj_simple = (color_adj=='Simple') or color_adj=='simple' + color_adj_keep_ratio = 0.5 + + if color_adj_flag and (ctype=='vision'): + x_adj = [] + for xi in x: + color_adj_f = color_adjust(ref_from=(xi+1)/2, ref_to=color_adj_to) + xi_adj = color_adj_f((xi+1)/2, keep=color_adj_keep_ratio, simple=color_adj_simple) + x_adj.append(xi_adj) + x = x_adj + else: + x = torch.clamp((x+1.0)/2.0, min=0.0, max=1.0) + x = [tvtrans.ToPILImage()(xi) for xi in x] + return x + + elif xtype == 'text': + prompt_temperature = 1.0 + prompt_merge_same_adj_word = True + x = net.optimus_decode(z, temperature=prompt_temperature) + if prompt_merge_same_adj_word: + xnew = [] + for xi in x: + xi_split = xi.split() + xinew = [] + for idxi, wi in enumerate(xi_split): + if idxi!=0 and wi==xi_split[idxi-1]: + continue + xinew.append(wi) + xnew.append(' '.join(xinew)) + x = xnew + return x + + def inference(self, xtype, cin, ctype, scale=7.5, n_samples=None, color_adj=None,): + net = self.net + sampler = self.sampler + ddim_steps = 50 + ddim_eta = 0.0 + + if xtype == 'image': + n_samples = n_sample_image_default if n_samples is None else n_samples + elif xtype == 'text': + n_samples = n_sample_text_default if n_samples is None else n_samples + + if ctype in ['prompt', 'text']: + c = net.clip_encode_text(n_samples * [cin]) + u = None + if scale != 1.0: + u = net.clip_encode_text(n_samples * [""]) + + elif ctype in ['vision', 'image']: + cin = self.regularize_image(cin) + ctemp = cin*2 - 1 + ctemp = ctemp[None].repeat(n_samples, 1, 1, 1) + c = net.clip_encode_vision(ctemp) + u = None + if scale != 1.0: + dummy = torch.zeros_like(ctemp) + u = net.clip_encode_vision(dummy) + + u, c = [u.half(), c.half()] if self.fp16 else [u, c] + + if xtype == 'image': + h, w = [512, 512] + shape = [n_samples, 4, h//8, w//8] + z, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=c, + unconditional_guidance_scale=scale, + unconditional_conditioning=u, + xtype=xtype, ctype=ctype, + eta=ddim_eta, + verbose=False,) + x = self.decode(z, xtype, ctype, color_adj=color_adj, color_adj_to=cin) + return x + + elif xtype == 'text': + n = 768 + shape = [n_samples, n] + z, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=c, + unconditional_guidance_scale=scale, + unconditional_conditioning=u, + xtype=xtype, ctype=ctype, + eta=ddim_eta, + verbose=False,) + x = self.decode(z, xtype, ctype) + return x + + def application_disensemble(self, cin, n_samples=None, level=0, color_adj=None,): + net = self.net + scale = 7.5 + sampler = self.sampler + ddim_steps = 50 + ddim_eta = 0.0 + n_samples = n_sample_image_default if n_samples is None else n_samples + + cin = self.regularize_image(cin) + ctemp = cin*2 - 1 + ctemp = ctemp[None].repeat(n_samples, 1, 1, 1) + c = net.clip_encode_vision(ctemp) + u = None + if scale != 1.0: + dummy = torch.zeros_like(ctemp) + u = net.clip_encode_vision(dummy) + u, c = [u.half(), c.half()] if self.fp16 else [u, c] + + if level == 0: + pass + else: + c_glb = c[:, 0:1] + c_loc = c[:, 1: ] + u_glb = u[:, 0:1] + u_loc = u[:, 1: ] + + if level == -1: + c_loc = self.remove_low_rank(c_loc, demean=True, q=50, q_remove=1) + u_loc = self.remove_low_rank(u_loc, demean=True, q=50, q_remove=1) + if level == -2: + c_loc = self.remove_low_rank(c_loc, demean=True, q=50, q_remove=2) + u_loc = self.remove_low_rank(u_loc, demean=True, q=50, q_remove=2) + if level == 1: + c_loc = self.find_low_rank(c_loc, demean=True, q=10) + u_loc = self.find_low_rank(u_loc, demean=True, q=10) + if level == 2: + c_loc = self.find_low_rank(c_loc, demean=True, q=2) + u_loc = self.find_low_rank(u_loc, demean=True, q=2) + + c = torch.cat([c_glb, c_loc], dim=1) + u = torch.cat([u_glb, u_loc], dim=1) + + h, w = [512, 512] + shape = [n_samples, 4, h//8, w//8] + z, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=c, + unconditional_guidance_scale=scale, + unconditional_conditioning=u, + xtype='image', ctype='vision', + eta=ddim_eta, + verbose=False,) + x = self.decode(z, 'image', 'vision', color_adj=color_adj, color_adj_to=cin) + return x + + def find_low_rank(self, x, demean=True, q=20, niter=10): + if demean: + x_mean = x.mean(-1, keepdim=True) + x_input = x - x_mean + else: + x_input = x + + if x_input.dtype == torch.float16: + fp16 = True + x_input = x_input.float() + else: + fp16 = False + + u, s, v = torch.pca_lowrank(x_input, q=q, center=False, niter=niter) + ss = torch.stack([torch.diag(si) for si in s]) + x_lowrank = torch.bmm(torch.bmm(u, ss), torch.permute(v, [0, 2, 1])) + + if fp16: + x_lowrank = x_lowrank.half() + + if demean: + x_lowrank += x_mean + return x_lowrank + + def remove_low_rank(self, x, demean=True, q=20, niter=10, q_remove=10): + if demean: + x_mean = x.mean(-1, keepdim=True) + x_input = x - x_mean + else: + x_input = x + + if x_input.dtype == torch.float16: + fp16 = True + x_input = x_input.float() + else: + fp16 = False + + u, s, v = torch.pca_lowrank(x_input, q=q, center=False, niter=niter) + s[:, 0:q_remove] = 0 + ss = torch.stack([torch.diag(si) for si in s]) + x_lowrank = torch.bmm(torch.bmm(u, ss), torch.permute(v, [0, 2, 1])) + + if fp16: + x_lowrank = x_lowrank.half() + + if demean: + x_lowrank += x_mean + return x_lowrank + + def application_dualguided(self, cim, ctx, n_samples=None, mixing=0.5, color_adj=None, ): + net = self.net + scale = 7.5 + sampler = self.sampler + ddim_steps = 50 + ddim_eta = 0.0 + n_samples = n_sample_image_default if n_samples is None else n_samples + + ctemp0 = self.regularize_image(cim) + ctemp1 = ctemp0*2 - 1 + ctemp1 = ctemp1[None].repeat(n_samples, 1, 1, 1) + cim = net.clip_encode_vision(ctemp1) + uim = None + if scale != 1.0: + dummy = torch.zeros_like(ctemp1) + uim = net.clip_encode_vision(dummy) + + ctx = net.clip_encode_text(n_samples * [ctx]) + utx = None + if scale != 1.0: + utx = net.clip_encode_text(n_samples * [""]) + + uim, cim = [uim.half(), cim.half()] if self.fp16 else [uim, cim] + utx, ctx = [utx.half(), ctx.half()] if self.fp16 else [utx, ctx] + + h, w = [512, 512] + shape = [n_samples, 4, h//8, w//8] + + z, _ = sampler.sample_dc( + steps=ddim_steps, + shape=shape, + first_conditioning=[uim, cim], + second_conditioning=[utx, ctx], + unconditional_guidance_scale=scale, + xtype='image', + first_ctype='vision', + second_ctype='prompt', + eta=ddim_eta, + verbose=False, + mixed_ratio=(1-mixing), ) + x = self.decode(z, 'image', 'vision', color_adj=color_adj, color_adj_to=ctemp0) + return x + + def application_i2t2i(self, cim, ctx_n, ctx_p, n_samples=None, color_adj=None,): + net = self.net + scale = 7.5 + sampler = self.sampler + ddim_steps = 50 + ddim_eta = 0.0 + prompt_temperature = 1.0 + n_samples = n_sample_image_default if n_samples is None else n_samples + + ctemp0 = self.regularize_image(cim) + ctemp1 = ctemp0*2 - 1 + ctemp1 = ctemp1[None].repeat(n_samples, 1, 1, 1) + cim = net.clip_encode_vision(ctemp1) + uim = None + if scale != 1.0: + dummy = torch.zeros_like(ctemp1) + uim = net.clip_encode_vision(dummy) + + uim, cim = [uim.half(), cim.half()] if self.fp16 else [uim, cim] + + n = 768 + shape = [n_samples, n] + zt, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=cim, + unconditional_guidance_scale=scale, + unconditional_conditioning=uim, + xtype='text', ctype='vision', + eta=ddim_eta, + verbose=False,) + ztn = net.optimus_encode([ctx_n]) + ztp = net.optimus_encode([ctx_p]) + + ztn_norm = ztn / ztn.norm(dim=1) + zt_proj_mag = torch.matmul(zt, ztn_norm[0]) + zt_perp = zt - zt_proj_mag[:, None] * ztn_norm + zt_newd = zt_perp + ztp + ctx_new = net.optimus_decode(zt_newd, temperature=prompt_temperature) + + ctx_new = net.clip_encode_text(ctx_new) + ctx_p = net.clip_encode_text([ctx_p]) + ctx_new = torch.cat([ctx_new, ctx_p.repeat(n_samples, 1, 1)], dim=1) + utx_new = net.clip_encode_text(n_samples * [""]) + utx_new = torch.cat([utx_new, utx_new], dim=1) + + cim_loc = cim[:, 1: ] + cim_loc_new = self.find_low_rank(cim_loc, demean=True, q=10) + cim_new = cim_loc_new + uim_new = uim[:, 1:] + + h, w = [512, 512] + shape = [n_samples, 4, h//8, w//8] + z, _ = sampler.sample_dc( + steps=ddim_steps, + shape=shape, + first_conditioning=[uim_new, cim_new], + second_conditioning=[utx_new, ctx_new], + unconditional_guidance_scale=scale, + xtype='image', + first_ctype='vision', + second_ctype='prompt', + eta=ddim_eta, + verbose=False, + mixed_ratio=0.33, ) + + x = self.decode(z, 'image', 'vision', color_adj=color_adj, color_adj_to=ctemp0) + return x + +def main(netwrapper, + app, + image=None, + prompt=None, + nprompt=None, + pprompt=None, + color_adj=None, + disentanglement_level=None, + dual_guided_mixing=None, + n_samples=4, + seed=0,): + + if seed is not None: + seed = 0 if seed<0 else seed + np.random.seed(seed) + torch.manual_seed(seed+100) + + if app == 'text-to-image': + print('Running [{}] with prompt [{}], n_samples [{}], seed [{}].'.format( + app, prompt, n_samples, seed)) + if (prompt is None) or (prompt == ""): + return None, None + with torch.no_grad(): + rv = netwrapper.inference( + xtype = 'image', + cin = prompt, + ctype = 'prompt', + n_samples = n_samples, ) + return rv, None + + elif app == 'image-variation': + print('Running [{}] with image [{}], color_adj [{}], n_samples [{}], seed [{}].'.format( + app, image, color_adj, n_samples, seed)) + if image is None: + return None, None + with torch.no_grad(): + rv = netwrapper.inference( + xtype = 'image', + cin = image, + ctype = 'vision', + color_adj = color_adj, + n_samples = n_samples, ) + return rv, None + + elif app == 'image-to-text': + print('Running [{}] with iamge [{}], n_samples [{}], seed [{}].'.format( + app, image, n_samples, seed)) + if image is None: + return None, None + with torch.no_grad(): + rv = netwrapper.inference( + xtype = 'text', + cin = image, + ctype = 'vision', + n_samples = n_samples, ) + return None, '\n'.join(rv) + + elif app == 'text-variation': + print('Running [{}] with prompt [{}], n_samples [{}], seed [{}].'.format( + app, prompt, n_samples, seed)) + if prompt is None: + return None, None + with torch.no_grad(): + rv = netwrapper.inference( + xtype = 'text', + cin = prompt, + ctype = 'prompt', + n_samples = n_samples, ) + return None, '\n'.join(rv) + + elif app == 'disentanglement': + print('Running [{}] with image [{}], color_adj [{}], disentanglement_level [{}], n_samples [{}], seed [{}].'.format( + app, image, color_adj, disentanglement_level, n_samples, seed)) + if image is None: + return None, None + with torch.no_grad(): + rv = netwrapper.application_disensemble( + cin = image, + level = disentanglement_level, + color_adj = color_adj, + n_samples = n_samples, ) + return rv, None + + elif app == 'dual-guided': + print('Running [{}] with image [{}], prompt [{}], color_adj [{}], dual_guided_mixing [{}], n_samples [{}], seed [{}].'.format( + app, image, prompt, color_adj, dual_guided_mixing, n_samples, seed)) + if (image is None) or (prompt is None) or (prompt==""): + return None, None + with torch.no_grad(): + rv = netwrapper.application_dualguided( + cim = image, + ctx = prompt, + mixing = dual_guided_mixing, + color_adj = color_adj, + n_samples = n_samples, ) + return rv, None + + elif app == 'i2t2i': + print('Running [{}] with image [{}], nprompt [{}], pprompt [{}], color_adj [{}], n_samples [{}], seed [{}].'.format( + app, image, nprompt, pprompt, color_adj, n_samples, seed)) + if (image is None) or (nprompt is None) or (nprompt=="") \ + or (pprompt is None) or (pprompt==""): + return None, None + with torch.no_grad(): + rv = netwrapper.application_i2t2i( + cim = image, + ctx_n = nprompt, + ctx_p = pprompt, + color_adj = color_adj, + n_samples = n_samples, ) + return rv, None + + else: + assert False, "No such mode!" + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "--app", type=str, default="text-to-image", + help="Choose the application from ["\ + "text-to-image, image-variation, "\ + "image-to-text, text-variation, "\ + "disentanglement, dual-guided, i2t2i]") + + parser.add_argument( + "--model", type=str, default="official", + help="Choose the model type from ["\ + "dc, official]") + + parser.add_argument( + "--prompt", type=str, + default="a dream of a village in china, by Caspar "\ + "David Friedrich, matte painting trending on artstation HQ") + + parser.add_argument("--image", type=str) + + parser.add_argument("--nprompt", type=str) + + parser.add_argument("--pprompt", type=str) + + parser.add_argument("--coloradj", type=str, default='simple') + + parser.add_argument("--dislevel", type=int, default=0) + + parser.add_argument("--dgmixing", type=float, default=0.7) + + parser.add_argument("--nsample", type=int, default=4) + + parser.add_argument("--seed", type=int) + + parser.add_argument("--save", type=str, default='log', + help="The path or file the result will save into") + + parser.add_argument("--gpu", type=int, default=0) + + parser.add_argument("--fp16", action="store_true") + + # parser.add_argument("--pth", type=str, default='pretrained/vd-four-flow-v1-0.pth') + + args = parser.parse_args() + + assert args.app in [ + "text-to-image", "image-variation", + "image-to-text", "text-variation", + "disentanglement", "dual-guided", "i2t2i"], \ + "Unknown app! Select from [text-to-image, image-variation, "\ + "image-to-text, text-variation, "\ + "disentanglement, dual-guided, i2t2i]" + + device=args.gpu if torch.cuda.is_available() else 'cpu' + + if args.model in ['4-flow', 'official']: + if args.fp16: + pth='pretrained/vd-four-flow-v1-0-fp16.pth' + else: + pth='pretrained/vd-four-flow-v1-0.pth' + vd_wrapper = vd_inference(pth=pth, fp16=args.fp16, device=device) + elif args.model in ['2-flow', 'dc']: + raise NotImplementedError + # vd_wrapper = vd_dc_inference(args.model, pth=args.pth, device=device) + elif args.model in ['1-flow', 'basic']: + raise NotImplementedError + # vd_wrapper = vd_basic_inference(args.model, pth=args.pth, device=device) + else: + assert False, "No such model! Select model from [4-flow(official), 2-flow(dc), 1-flow(basic)]" + + imout, txtout = main( + netwrapper=vd_wrapper, + app=args.app, + image=args.image, + prompt=args.prompt, + nprompt=args.nprompt, + pprompt=args.pprompt, + color_adj=args.coloradj, + disentanglement_level=args.dislevel, + dual_guided_mixing=args.dgmixing, + n_samples=args.nsample, + seed=args.seed,) + + if imout is not None: + imout = auto_merge_imlist([np.array(i) for i in imout]) + imout = PIL.Image.fromarray(imout) + if osp.isdir(args.save): + imout.save(osp.join(args.save, 'imout.png')) + print('Output image saved to {}.'.format(osp.join(args.save, 'imout.png'))) + else: + imout.save(osp.join(args.save)) + print('Output image saved to {}.'.format(args.save)) + + if txtout is not None: + print(txtout) diff --git a/versatile_diffusion/lib/__init__.py b/versatile_diffusion/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/versatile_diffusion/lib/__pycache__/__init__.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07dd29bda2e0a71d6f37ceef61ccd1288f3f6222 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/__init__.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/__init__.cpython-38.pyc b/versatile_diffusion/lib/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78d5b5055dffec24260d365a8aad29601086516b Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/__init__.cpython-38.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24980d2665483faad50cf9a271c8cf2b3e70a497 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-38.pyc b/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f5fb0f076740651302d2eb5c11e275310c9c4cd Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/cfg_helper.cpython-38.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c3c89fe1a07445501eddb44e3a61482c2cac7e6 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-38.pyc b/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8be5360639ff2d733cbc6514ef760aaf5396e703 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/cfg_holder.cpython-38.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/log_service.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/log_service.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1aaaf575a725dfb2c9632f77bcab144a7cfb6c5 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/log_service.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/sync.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/sync.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a5055bcd9c9cd939c9e8d5d0168ca5c4cc21e05 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/sync.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/sync.cpython-38.pyc b/versatile_diffusion/lib/__pycache__/sync.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be4ec56df943880f60fce06e5544e0571de22225 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/sync.cpython-38.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/utils.cpython-310.pyc b/versatile_diffusion/lib/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e9b70ec9b588c6940476d8454988696df1c30b5 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/utils.cpython-310.pyc differ diff --git a/versatile_diffusion/lib/__pycache__/utils.cpython-38.pyc b/versatile_diffusion/lib/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4523d0070413ab42b92ff071bfbd91e6be803874 Binary files /dev/null and b/versatile_diffusion/lib/__pycache__/utils.cpython-38.pyc differ diff --git a/versatile_diffusion/lib/cfg_helper.py b/versatile_diffusion/lib/cfg_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..6d93c2175b78216397baf153ce8f131d3ee6a619 --- /dev/null +++ b/versatile_diffusion/lib/cfg_helper.py @@ -0,0 +1,664 @@ +import os +import os.path as osp +import shutil +import copy +import time +import pprint +import numpy as np +import torch +import matplotlib +import argparse +import json +import yaml +from easydict import EasyDict as edict + +from .model_zoo import get_model + +############ +# cfg_bank # +############ + +def cfg_solvef(cmd, root): + if not isinstance(cmd, str): + return cmd + + if cmd.find('SAME')==0: + zoom = root + p = cmd[len('SAME'):].strip('()').split('.') + p = [pi.strip() for pi in p] + for pi in p: + try: + pi = int(pi) + except: + pass + + try: + zoom = zoom[pi] + except: + return cmd + return cfg_solvef(zoom, root) + + if cmd.find('SEARCH')==0: + zoom = root + p = cmd[len('SEARCH'):].strip('()').split('.') + p = [pi.strip() for pi in p] + find = True + # Depth first search + for pi in p: + try: + pi = int(pi) + except: + pass + + try: + zoom = zoom[pi] + except: + find = False + break + + if find: + return cfg_solvef(zoom, root) + else: + if isinstance(root, dict): + for ri in root: + rv = cfg_solvef(cmd, root[ri]) + if rv != cmd: + return rv + if isinstance(root, list): + for ri in root: + rv = cfg_solvef(cmd, ri) + if rv != cmd: + return rv + return cmd + + if cmd.find('MODEL')==0: + goto = cmd[len('MODEL'):].strip('()') + return model_cfg_bank()(goto) + + if cmd.find('DATASET')==0: + goto = cmd[len('DATASET'):].strip('()') + return dataset_cfg_bank()(goto) + + return cmd + +def cfg_solve(cfg, cfg_root): + # The function solve cfg element such that + # all sorrogate input are settled. + # (i.e. SAME(***) ) + if isinstance(cfg, list): + for i in range(len(cfg)): + if isinstance(cfg[i], (list, dict)): + cfg[i] = cfg_solve(cfg[i], cfg_root) + else: + cfg[i] = cfg_solvef(cfg[i], cfg_root) + if isinstance(cfg, dict): + for k in cfg: + if isinstance(cfg[k], (list, dict)): + cfg[k] = cfg_solve(cfg[k], cfg_root) + else: + cfg[k] = cfg_solvef(cfg[k], cfg_root) + return cfg + +class model_cfg_bank(object): + def __init__(self): + self.cfg_dir = osp.join('versatile_diffusion/configs', 'model') + self.cfg_bank = edict() + + def __call__(self, name): + if name not in self.cfg_bank: + cfg_path = self.get_yaml_path(name) + with open(cfg_path, 'r') as f: + cfg_new = yaml.load( + f, Loader=yaml.FullLoader) + cfg_new = edict(cfg_new) + self.cfg_bank.update(cfg_new) + + cfg = self.cfg_bank[name] + cfg.name = name + if 'super_cfg' not in cfg: + cfg = cfg_solve(cfg, cfg) + self.cfg_bank[name] = cfg + return copy.deepcopy(cfg) + + super_cfg = self.__call__(cfg.super_cfg) + # unlike other field, + # args will not be replaced but update. + if 'args' in cfg: + if 'args' in super_cfg: + super_cfg.args.update(cfg.args) + else: + super_cfg.args = cfg.args + cfg.pop('args') + + super_cfg.update(cfg) + super_cfg.pop('super_cfg') + cfg = super_cfg + try: + delete_args = cfg.pop('delete_args') + except: + delete_args = [] + + for dargs in delete_args: + cfg.args.pop(dargs) + + cfg = cfg_solve(cfg, cfg) + self.cfg_bank[name] = cfg + return copy.deepcopy(cfg) + + def get_yaml_path(self, name): + if name.find('ldm')==0: + return osp.join( + self.cfg_dir, 'ldm.yaml') + elif name.find('comodgan')==0: + return osp.join( + self.cfg_dir, 'comodgan.yaml') + elif name.find('stylegan')==0: + return osp.join( + self.cfg_dir, 'stylegan.yaml') + elif name.find('absgan')==0: + return osp.join( + self.cfg_dir, 'absgan.yaml') + elif name.find('ashgan')==0: + return osp.join( + self.cfg_dir, 'ashgan.yaml') + elif name.find('sr3')==0: + return osp.join( + self.cfg_dir, 'sr3.yaml') + elif name.find('specdiffsr')==0: + return osp.join( + self.cfg_dir, 'specdiffsr.yaml') + elif name.find('openai_unet')==0: + return osp.join( + self.cfg_dir, 'openai_unet.yaml') + elif name.find('clip')==0: + return osp.join( + self.cfg_dir, 'clip.yaml') + elif name.find('sd')==0: + return osp.join( + self.cfg_dir, 'sd.yaml') + elif name.find('vd')==0: + return osp.join( + self.cfg_dir, 'vd.yaml') + elif name.find('optimus')==0: + return osp.join( + self.cfg_dir, 'optimus.yaml') + else: + raise ValueError + +class dataset_cfg_bank(object): + def __init__(self): + self.cfg_dir = osp.join('configs', 'dataset') + self.cfg_bank = edict() + + def __call__(self, name): + if name not in self.cfg_bank: + cfg_path = self.get_yaml_path(name) + with open(cfg_path, 'r') as f: + cfg_new = yaml.load( + f, Loader=yaml.FullLoader) + cfg_new = edict(cfg_new) + self.cfg_bank.update(cfg_new) + + cfg = self.cfg_bank[name] + cfg.name = name + if cfg.get('super_cfg', None) is None: + cfg = cfg_solve(cfg, cfg) + self.cfg_bank[name] = cfg + return copy.deepcopy(cfg) + + super_cfg = self.__call__(cfg.super_cfg) + super_cfg.update(cfg) + cfg = super_cfg + cfg.super_cfg = None + try: + delete = cfg.pop('delete') + except: + delete = [] + + for dargs in delete: + cfg.pop(dargs) + + cfg = cfg_solve(cfg, cfg) + self.cfg_bank[name] = cfg + return copy.deepcopy(cfg) + + def get_yaml_path(self, name): + if name.find('cityscapes')==0: + return osp.join( + self.cfg_dir, 'cityscapes.yaml') + elif name.find('div2k')==0: + return osp.join( + self.cfg_dir, 'div2k.yaml') + elif name.find('gandiv2k')==0: + return osp.join( + self.cfg_dir, 'gandiv2k.yaml') + elif name.find('srbenchmark')==0: + return osp.join( + self.cfg_dir, 'srbenchmark.yaml') + elif name.find('imagedir')==0: + return osp.join( + self.cfg_dir, 'imagedir.yaml') + elif name.find('places2')==0: + return osp.join( + self.cfg_dir, 'places2.yaml') + elif name.find('ffhq')==0: + return osp.join( + self.cfg_dir, 'ffhq.yaml') + elif name.find('imcpt')==0: + return osp.join( + self.cfg_dir, 'imcpt.yaml') + elif name.find('texture')==0: + return osp.join( + self.cfg_dir, 'texture.yaml') + elif name.find('openimages')==0: + return osp.join( + self.cfg_dir, 'openimages.yaml') + elif name.find('laion2b')==0: + return osp.join( + self.cfg_dir, 'laion2b.yaml') + elif name.find('laionart')==0: + return osp.join( + self.cfg_dir, 'laionart.yaml') + elif name.find('celeba')==0: + return osp.join( + self.cfg_dir, 'celeba.yaml') + elif name.find('coyo')==0: + return osp.join( + self.cfg_dir, 'coyo.yaml') + elif name.find('pafc')==0: + return osp.join( + self.cfg_dir, 'pafc.yaml') + elif name.find('coco')==0: + return osp.join( + self.cfg_dir, 'coco.yaml') + else: + raise ValueError + +class experiment_cfg_bank(object): + def __init__(self): + self.cfg_dir = osp.join('configs', 'experiment') + self.cfg_bank = edict() + + def __call__(self, name): + if name not in self.cfg_bank: + cfg_path = self.get_yaml_path(name) + with open(cfg_path, 'r') as f: + cfg = yaml.load( + f, Loader=yaml.FullLoader) + cfg = edict(cfg) + + cfg = cfg_solve(cfg, cfg) + cfg = cfg_solve(cfg, cfg) + # twice for SEARCH + self.cfg_bank[name] = cfg + return copy.deepcopy(cfg) + + def get_yaml_path(self, name): + return osp.join( + self.cfg_dir, name+'.yaml') + +def load_cfg_yaml(path): + if osp.isfile(path): + cfg_path = path + elif osp.isfile(osp.join('configs', 'experiment', path)): + cfg_path = osp.join('configs', 'experiment', path) + elif osp.isfile(osp.join('configs', 'experiment', path+'.yaml')): + cfg_path = osp.join('configs', 'experiment', path+'.yaml') + else: + assert False, 'No such config!' + + with open(cfg_path, 'r') as f: + cfg = yaml.load(f, Loader=yaml.FullLoader) + cfg = edict(cfg) + cfg = cfg_solve(cfg, cfg) + cfg = cfg_solve(cfg, cfg) + return cfg + +############## +# cfg_helper # +############## + +def get_experiment_id(ref=None): + if ref is None: + time.sleep(0.5) + return int(time.time()*100) + else: + try: + return int(ref) + except: + pass + + _, ref = osp.split(ref) + ref = ref.split('_')[0] + try: + return int(ref) + except: + assert False, 'Invalid experiment ID!' + +def record_resume_cfg(path): + cnt = 0 + while True: + if osp.exists(path+'.{:04d}'.format(cnt)): + cnt += 1 + continue + shutil.copyfile(path, path+'.{:04d}'.format(cnt)) + break + +def get_command_line_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--debug', action='store_true', default=False) + parser.add_argument('--config', type=str) + parser.add_argument('--gpu', nargs='+', type=int) + + parser.add_argument('--node_rank', type=int, default=0) + parser.add_argument('--nodes', type=int, default=1) + parser.add_argument('--addr', type=str, default='127.0.0.1') + parser.add_argument('--port', type=int, default=11233) + + parser.add_argument('--signature', nargs='+', type=str) + parser.add_argument('--seed', type=int) + + parser.add_argument('--eval', type=str) + parser.add_argument('--eval_subdir', type=str) + parser.add_argument('--pretrained', type=str) + + parser.add_argument('--resume_dir', type=str) + parser.add_argument('--resume_step', type=int) + parser.add_argument('--resume_weight', type=str) + + args = parser.parse_args() + + # Special handling the resume + if args.resume_dir is not None: + cfg = edict() + cfg.env = edict() + cfg.env.debug = args.debug + cfg.env.resume = edict() + cfg.env.resume.dir = args.resume_dir + cfg.env.resume.step = args.resume_step + cfg.env.resume.weight = args.resume_weight + return cfg + + cfg = load_cfg_yaml(args.config) + cfg.env.debug = args.debug + cfg.env.gpu_device = [0] if args.gpu is None else list(args.gpu) + cfg.env.master_addr = args.addr + cfg.env.master_port = args.port + cfg.env.dist_url = 'tcp://{}:{}'.format(args.addr, args.port) + cfg.env.node_rank = args.node_rank + cfg.env.nodes = args.nodes + + istrain = False if args.eval is not None else True + isdebug = cfg.env.debug + + if istrain: + if isdebug: + cfg.env.experiment_id = 999999999999 + cfg.train.signature = ['debug'] + else: + cfg.env.experiment_id = get_experiment_id() + if args.signature is not None: + cfg.train.signature = args.signature + else: + if 'train' in cfg: + cfg.pop('train') + cfg.env.experiment_id = get_experiment_id(args.eval) + if args.signature is not None: + cfg.eval.signature = args.signature + + if isdebug and (args.eval is None): + cfg.env.experiment_id = 999999999999 + cfg.eval.signature = ['debug'] + + if args.eval_subdir is not None: + if isdebug: + cfg.eval.eval_subdir = 'debug' + else: + cfg.eval.eval_subdir = args.eval_subdir + if args.pretrained is not None: + cfg.eval.pretrained = args.pretrained + # The override pretrained over the setting in cfg.model + + if args.seed is not None: + cfg.env.rnd_seed = args.seed + + return cfg + +def cfg_initiates(cfg): + cfge = cfg.env + isdebug = cfge.debug + isresume = 'resume' in cfge + istrain = 'train' in cfg + haseval = 'eval' in cfg + cfgt = cfg.train if istrain else None + cfgv = cfg.eval if haseval else None + + ############################### + # get some environment params # + ############################### + + cfge.computer = os.uname() + cfge.torch_version = str(torch.__version__) + + ########## + # resume # + ########## + + if isresume: + resume_cfg_path = osp.join(cfge.resume.dir, 'config.yaml') + record_resume_cfg(resume_cfg_path) + with open(resume_cfg_path, 'r') as f: + cfg_resume = yaml.load(f, Loader=yaml.FullLoader) + cfg_resume = edict(cfg_resume) + cfg_resume.env.update(cfge) + cfg = cfg_resume + cfge = cfg.env + log_file = cfg.train.log_file + + print('') + print('##########') + print('# resume #') + print('##########') + print('') + with open(log_file, 'a') as f: + print('', file=f) + print('##########', file=f) + print('# resume #', file=f) + print('##########', file=f) + print('', file=f) + + pprint.pprint(cfg) + with open(log_file, 'a') as f: + pprint.pprint(cfg, f) + + #################### + # node distributed # + #################### + + if cfg.env.master_addr!='127.0.0.1': + os.environ['MASTER_ADDR'] = cfge.master_addr + os.environ['MASTER_PORT'] = '{}'.format(cfge.master_port) + if cfg.env.dist_backend=='nccl': + os.environ['NCCL_SOCKET_FAMILY'] = 'AF_INET' + if cfg.env.dist_backend=='gloo': + os.environ['GLOO_SOCKET_FAMILY'] = 'AF_INET' + + ####################### + # cuda visible device # + ####################### + + os.environ["CUDA_VISIBLE_DEVICES"] = ','.join( + [str(gid) for gid in cfge.gpu_device]) + + ##################### + # return resume cfg # + ##################### + + if isresume: + return cfg + + ############################################# + # some misc setting that not need in resume # + ############################################# + + cfgm = cfg.model + cfge.gpu_count = len(cfge.gpu_device) + + ########################################## + # align batch size and num worker config # + ########################################## + + gpu_n = cfge.gpu_count * cfge.nodes + def align_batch_size(bs, bs_per_gpu): + assert (bs is not None) or (bs_per_gpu is not None) + bs = bs_per_gpu * gpu_n if bs is None else bs + bs_per_gpu = bs // gpu_n if bs_per_gpu is None else bs_per_gpu + assert (bs == bs_per_gpu * gpu_n) + return bs, bs_per_gpu + + if istrain: + cfgt.batch_size, cfgt.batch_size_per_gpu = \ + align_batch_size(cfgt.batch_size, cfgt.batch_size_per_gpu) + cfgt.dataset_num_workers, cfgt.dataset_num_workers_per_gpu = \ + align_batch_size(cfgt.dataset_num_workers, cfgt.dataset_num_workers_per_gpu) + if haseval: + cfgv.batch_size, cfgv.batch_size_per_gpu = \ + align_batch_size(cfgv.batch_size, cfgv.batch_size_per_gpu) + cfgv.dataset_num_workers, cfgv.dataset_num_workers_per_gpu = \ + align_batch_size(cfgv.dataset_num_workers, cfgv.dataset_num_workers_per_gpu) + + ################## + # create log dir # + ################## + + if istrain: + if not isdebug: + sig = cfgt.get('signature', []) + version = get_model().get_version(cfgm.type) + sig = sig + ['v{}'.format(version), 's{}'.format(cfge.rnd_seed)] + else: + sig = ['debug'] + + log_dir = [ + cfge.log_root_dir, + '{}_{}'.format(cfgm.symbol, cfgt.dataset.symbol), + '_'.join([str(cfge.experiment_id)] + sig) + ] + log_dir = osp.join(*log_dir) + log_file = osp.join(log_dir, 'train.log') + if not osp.exists(log_file): + os.makedirs(osp.dirname(log_file)) + cfgt.log_dir = log_dir + cfgt.log_file = log_file + + if haseval: + cfgv.log_dir = log_dir + cfgv.log_file = log_file + else: + model_symbol = cfgm.symbol + if cfgv.get('dataset', None) is None: + dataset_symbol = 'nodataset' + else: + dataset_symbol = cfgv.dataset.symbol + + log_dir = osp.join(cfge.log_root_dir, '{}_{}'.format(model_symbol, dataset_symbol)) + exp_dir = search_experiment_folder(log_dir, cfge.experiment_id) + if exp_dir is None: + if not isdebug: + sig = cfgv.get('signature', []) + ['evalonly'] + else: + sig = ['debug'] + exp_dir = '_'.join([str(cfge.experiment_id)] + sig) + + eval_subdir = cfgv.get('eval_subdir', None) + # override subdir in debug mode (if eval_subdir is set) + eval_subdir = 'debug' if (eval_subdir is not None) and isdebug else eval_subdir + + if eval_subdir is not None: + log_dir = osp.join(log_dir, exp_dir, eval_subdir) + else: + log_dir = osp.join(log_dir, exp_dir) + + disable_log_override = cfgv.get('disable_log_override', False) + if osp.isdir(log_dir): + if disable_log_override: + assert False, 'Override an exsited log_dir is disabled at [{}]'.format(log_dir) + else: + os.makedirs(log_dir) + + log_file = osp.join(log_dir, 'eval.log') + cfgv.log_dir = log_dir + cfgv.log_file = log_file + + ###################### + # print and save cfg # + ###################### + + pprint.pprint(cfg) + with open(log_file, 'w') as f: + pprint.pprint(cfg, f) + with open(osp.join(log_dir, 'config.yaml'), 'w') as f: + yaml.dump(edict_2_dict(cfg), f) + + ############# + # save code # + ############# + + save_code = False + if istrain: + save_code = cfgt.get('save_code', False) + elif haseval: + save_code = cfgv.get('save_code', False) + + if save_code: + codedir = osp.join(log_dir, 'code') + if osp.exists(codedir): + shutil.rmtree(codedir) + for d in ['configs', 'lib']: + fromcodedir = d + tocodedir = osp.join(codedir, d) + shutil.copytree( + fromcodedir, tocodedir, + ignore=shutil.ignore_patterns( + '*__pycache__*', '*build*')) + for codei in os.listdir('.'): + if osp.splitext(codei)[1] == 'py': + shutil.copy(codei, codedir) + + ####################### + # set matplotlib mode # + ####################### + + if 'matplotlib_mode' in cfge: + try: + matplotlib.use(cfge.matplotlib_mode) + except: + print('Warning: matplotlib mode [{}] failed to be set!'.format(cfge.matplotlib_mode)) + + return cfg + +def edict_2_dict(x): + if isinstance(x, dict): + xnew = {} + for k in x: + xnew[k] = edict_2_dict(x[k]) + return xnew + elif isinstance(x, list): + xnew = [] + for i in range(len(x)): + xnew.append( edict_2_dict(x[i]) ) + return xnew + else: + return x + +def search_experiment_folder(root, exid): + target = None + for fi in os.listdir(root): + if not osp.isdir(osp.join(root, fi)): + continue + if int(fi.split('_')[0]) == exid: + if target is not None: + return None # duplicated + elif target is None: + target = fi + return target diff --git a/versatile_diffusion/lib/cfg_holder.py b/versatile_diffusion/lib/cfg_holder.py new file mode 100644 index 0000000000000000000000000000000000000000..a5cf16c4116931aef32a7275a63965a0d5f23ec7 --- /dev/null +++ b/versatile_diffusion/lib/cfg_holder.py @@ -0,0 +1,28 @@ +import copy + +def singleton(class_): + instances = {} + def getinstance(*args, **kwargs): + if class_ not in instances: + instances[class_] = class_(*args, **kwargs) + return instances[class_] + return getinstance + +############## +# cfg_holder # +############## + +@singleton +class cfg_unique_holder(object): + def __init__(self): + self.cfg = None + # this is use to track the main codes. + self.code = set() + def save_cfg(self, cfg): + self.cfg = copy.deepcopy(cfg) + def add_code(self, code): + """ + A new main code is reached and + its name is added. + """ + self.code.add(code) diff --git a/versatile_diffusion/lib/data_factory/__init__.py b/versatile_diffusion/lib/data_factory/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f0f7664990b2ea8954e0960f1be3682bc35d708a --- /dev/null +++ b/versatile_diffusion/lib/data_factory/__init__.py @@ -0,0 +1,6 @@ +from .common.ds_base import collate, get_dataset +from .common.ds_loader import get_loader +from .common.ds_transform import get_transform +from .common.ds_estimator import get_estimator +from .common.ds_formatter import get_formatter +from .common.ds_sampler import get_sampler diff --git a/versatile_diffusion/lib/log_service.py b/versatile_diffusion/lib/log_service.py new file mode 100644 index 0000000000000000000000000000000000000000..348afd412a2686a264a108fe6bf9e30e289d5947 --- /dev/null +++ b/versatile_diffusion/lib/log_service.py @@ -0,0 +1,166 @@ +import timeit +import numpy as np +import os +import os.path as osp +import shutil +import copy +import torch +import torch.nn as nn +import torch.distributed as dist +from .cfg_holder import cfg_unique_holder as cfguh +from . import sync + +print_console_local_rank0_only = True + +def print_log(*console_info): + local_rank = sync.get_rank('local') + if print_console_local_rank0_only and (local_rank!=0): + return + console_info = [str(i) for i in console_info] + console_info = ' '.join(console_info) + print(console_info) + + if local_rank!=0: + return + + log_file = None + try: + log_file = cfguh().cfg.train.log_file + except: + try: + log_file = cfguh().cfg.eval.log_file + except: + return + if log_file is not None: + with open(log_file, 'a') as f: + f.write(console_info + '\n') + +class distributed_log_manager(object): + def __init__(self): + self.sum = {} + self.cnt = {} + self.time_check = timeit.default_timer() + + cfgt = cfguh().cfg.train + use_tensorboard = getattr(cfgt, 'log_tensorboard', False) + + self.ddp = sync.is_ddp() + self.rank = sync.get_rank('local') + self.world_size = sync.get_world_size('local') + + self.tb = None + if use_tensorboard and (self.rank==0): + import tensorboardX + monitoring_dir = osp.join(cfguh().cfg.train.log_dir, 'tensorboard') + self.tb = tensorboardX.SummaryWriter(osp.join(monitoring_dir)) + + def accumulate(self, n, **data): + if n < 0: + raise ValueError + + for itemn, di in data.items(): + if itemn in self.sum: + self.sum[itemn] += di * n + self.cnt[itemn] += n + else: + self.sum[itemn] = di * n + self.cnt[itemn] = n + + def get_mean_value_dict(self): + value_gather = [ + self.sum[itemn]/self.cnt[itemn] \ + for itemn in sorted(self.sum.keys()) ] + + value_gather_tensor = torch.FloatTensor(value_gather).to(self.rank) + if self.ddp: + dist.all_reduce(value_gather_tensor, op=dist.ReduceOp.SUM) + value_gather_tensor /= self.world_size + + mean = {} + for idx, itemn in enumerate(sorted(self.sum.keys())): + mean[itemn] = value_gather_tensor[idx].item() + return mean + + def tensorboard_log(self, step, data, mode='train', **extra): + if self.tb is None: + return + if mode == 'train': + self.tb.add_scalar('other/epochn', extra['epochn'], step) + if 'lr' in extra: + self.tb.add_scalar('other/lr', extra['lr'], step) + for itemn, di in data.items(): + if itemn.find('loss') == 0: + self.tb.add_scalar('loss/'+itemn, di, step) + elif itemn == 'Loss': + self.tb.add_scalar('Loss', di, step) + else: + self.tb.add_scalar('other/'+itemn, di, step) + elif mode == 'eval': + if isinstance(data, dict): + for itemn, di in data.items(): + self.tb.add_scalar('eval/'+itemn, di, step) + else: + self.tb.add_scalar('eval', data, step) + return + + def train_summary(self, itern, epochn, samplen, lr, tbstep=None): + console_info = [ + 'Iter:{}'.format(itern), + 'Epoch:{}'.format(epochn), + 'Sample:{}'.format(samplen),] + + if lr is not None: + console_info += ['LR:{:.4E}'.format(lr)] + + mean = self.get_mean_value_dict() + + tbstep = itern if tbstep is None else tbstep + self.tensorboard_log( + tbstep, mean, mode='train', + itern=itern, epochn=epochn, lr=lr) + + loss = mean.pop('Loss') + mean_info = ['Loss:{:.4f}'.format(loss)] + [ + '{}:{:.4f}'.format(itemn, mean[itemn]) \ + for itemn in sorted(mean.keys()) \ + if itemn.find('loss') == 0 + ] + console_info += mean_info + console_info.append('Time:{:.2f}s'.format( + timeit.default_timer() - self.time_check)) + return ' , '.join(console_info) + + def clear(self): + self.sum = {} + self.cnt = {} + self.time_check = timeit.default_timer() + + def tensorboard_close(self): + if self.tb is not None: + self.tb.close() + +# ----- also include some small utils ----- + +def torch_to_numpy(*argv): + if len(argv) > 1: + data = list(argv) + else: + data = argv[0] + + if isinstance(data, torch.Tensor): + return data.to('cpu').detach().numpy() + + elif isinstance(data, (list, tuple)): + out = [] + for di in data: + out.append(torch_to_numpy(di)) + return out + + elif isinstance(data, dict): + out = {} + for ni, di in data.items(): + out[ni] = torch_to_numpy(di) + return out + + else: + return data diff --git a/versatile_diffusion/lib/sync.py b/versatile_diffusion/lib/sync.py new file mode 100644 index 0000000000000000000000000000000000000000..668672eea2312d8cdb4e3e7e73d383e19ac8fb4f --- /dev/null +++ b/versatile_diffusion/lib/sync.py @@ -0,0 +1,210 @@ +from multiprocessing import shared_memory +# import multiprocessing +# if hasattr(multiprocessing, "shared_memory"): +# from multiprocessing import shared_memory +# else: +# # workaround for single gpu inference on colab +# shared_memory = None + +import random +import pickle +import time +import copy +import torch +import torch.distributed as dist +from lib.cfg_holder import cfg_unique_holder as cfguh + +def singleton(class_): + instances = {} + def getinstance(*args, **kwargs): + if class_ not in instances: + instances[class_] = class_(*args, **kwargs) + return instances[class_] + return getinstance + +def is_ddp(): + return dist.is_available() and dist.is_initialized() + +def get_rank(type='local'): + ddp = is_ddp() + global_rank = dist.get_rank() if ddp else 0 + local_world_size = torch.cuda.device_count() + if type == 'global': + return global_rank + elif type == 'local': + return global_rank % local_world_size + elif type == 'node': + return global_rank // local_world_size + elif type == 'all': + return global_rank, \ + global_rank % local_world_size, \ + global_rank // local_world_size + else: + assert False, 'Unknown type' + +def get_world_size(type='local'): + ddp = is_ddp() + global_rank = dist.get_rank() if ddp else 0 + global_world_size = dist.get_world_size() if ddp else 1 + local_world_size = torch.cuda.device_count() + if type == 'global': + return global_world_size + elif type == 'local': + return local_world_size + elif type == 'node': + return global_world_size // local_world_size + elif type == 'all': + return global_world_size, local_world_size, \ + global_world_size // local_world_size + else: + assert False, 'Unknown type' + +class barrier_lock(object): + def __init__(self, n): + self.n = n + id = int(random.random()*10000) + int(time.time())*10000 + self.lock_shmname = 'barrier_lock_{}'.format(id) + lock_shm = shared_memory.SharedMemory( + name=self.lock_shmname, create=True, size=n) + for i in range(n): + lock_shm.buf[i] = 0 + lock_shm.close() + + def destroy(self): + try: + lock_shm = shared_memory.SharedMemory( + name=self.lock_shmname) + lock_shm.close() + lock_shm.unlink() + except: + return + + def wait(self, k): + lock_shm = shared_memory.SharedMemory( + name=self.lock_shmname) + assert lock_shm.buf[k] == 0, 'Two waits on the same id is not allowed.' + lock_shm.buf[k] = 1 + if k == 0: + while sum([lock_shm.buf[i]==0 for i in range(self.n)]) != 0: + pass + for i in range(self.n): + lock_shm.buf[i] = 0 + return + else: + while lock_shm.buf[k] != 0: + pass + +class nodewise_sync_global(object): + """ + This is the global part of nodewise_sync that need to call at master process + before spawn. + """ + def __init__(self): + self.local_world_size = get_world_size('local') + self.b_lock = barrier_lock(self.local_world_size) + id = int(random.random()*10000) + int(time.time())*10000 + self.id_shmname = 'nodewise_sync_id_shm_{}'.format(id) + + def destroy(self): + self.b_lock.destroy() + try: + shm = shared_memory.SharedMemory(name=self.id_shmname) + shm.close() + shm.unlink() + except: + return + +@singleton +class nodewise_sync(object): + """ + A class that centralize nodewise sync activities. + The backend is multiprocess sharememory, not torch, as torch not support this. + """ + def __init__(self): + pass + + def copy_global(self, reference): + self.local_world_size = reference.local_world_size + self.b_lock = reference.b_lock + self.id_shmname = reference.id_shmname + return self + + def local_init(self): + self.ddp = is_ddp() + self.global_rank, self.local_rank, self.node_rank = get_rank('all') + self.global_world_size, self.local_world_size, self.nodes = get_world_size('all') + if self.local_rank == 0: + temp = int(random.random()*10000) + int(time.time())*10000 + temp = pickle.dumps(temp) + shm = shared_memory.SharedMemory( + name=self.id_shmname, create=True, size=len(temp)) + shm.close() + return self + + def random_sync_id(self): + assert self.local_rank is not None, 'Not initialized!' + if self.local_rank == 0: + sync_id = int(random.random()*10000) + int(time.time())*10000 + data = pickle.dumps(sync_id) + shm = shared_memory.SharedMemory(name=self.id_shmname) + shm.buf[0:len(data)] = data[0:len(data)] + self.barrier() + shm.close() + else: + self.barrier() + shm = shared_memory.SharedMemory(name=self.id_shmname) + sync_id = pickle.loads(shm.buf) + shm.close() + return sync_id + + def barrier(self): + self.b_lock.wait(self.local_rank) + + def broadcast_r0(self, data=None): + assert self.local_rank is not None, 'Not initialized!' + id = self.random_sync_id() + shmname = 'broadcast_r0_{}'.format(id) + if self.local_rank == 0: + assert data!=None, 'Rank 0 needs to input data!' + data = pickle.dumps(data) + datan = len(data) + load_info_shm = shared_memory.SharedMemory( + name=shmname, create=True, size=datan) + load_info_shm.buf[0:datan] = data[0:datan] + self.barrier() + self.barrier() + load_info_shm.close() + load_info_shm.unlink() + return None + else: + assert data==None, 'Rank other than 1 should input None as data!' + self.barrier() + shm = shared_memory.SharedMemory(name=shmname) + data = pickle.loads(shm.buf) + shm.close() + self.barrier() + return data + + def destroy(self): + self.barrier.destroy() + try: + shm = shared_memory.SharedMemory(name=self.id_shmname) + shm.close() + shm.unlink() + except: + return + +# import contextlib + +# @contextlib.contextmanager +# def weight_sync(module, sync): +# assert isinstance(module, torch.nn.Module) +# if sync or not isinstance(module, torch.nn.parallel.DistributedDataParallel): +# yield +# else: +# with module.no_sync(): +# yield + +# def weight_sync(net): +# for parameters in net.parameters(): +# dist.all_reduce(parameters, dist.ReduceOp.AVG) \ No newline at end of file diff --git a/versatile_diffusion/lib/utils.py b/versatile_diffusion/lib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..51a5671e8a0e7669a3d6dd73fc6938ce698ae352 --- /dev/null +++ b/versatile_diffusion/lib/utils.py @@ -0,0 +1,635 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +# cudnn.enabled = True +# cudnn.benchmark = True +import torch.distributed as dist +import torch.multiprocessing as mp + +import os +import os.path as osp +import sys +import numpy as np +import pprint +import timeit +import time +import copy +import matplotlib.pyplot as plt + +from .cfg_holder import cfg_unique_holder as cfguh + +from .data_factory import \ + get_dataset, collate, \ + get_loader, \ + get_transform, \ + get_estimator, \ + get_formatter, \ + get_sampler + +from .model_zoo import \ + get_model, get_optimizer, get_scheduler + +from .log_service import print_log, distributed_log_manager + +from .evaluator import get_evaluator +from . import sync + +class train_stage(object): + """ + This is a template for a train stage, + (can be either train or test or anything) + Usually, it takes RANK + one dataloader, one model, one optimizer, one scheduler. + But it is not limited to these parameters. + """ + def __init__(self): + self.nested_eval_stage = None + self.rv_keep = None + + def is_better(self, x): + return (self.rv_keep is None) or (x>self.rv_keep) + + def set_model(self, net, mode): + if mode == 'train': + return net.train() + elif mode == 'eval': + return net.eval() + else: + raise ValueError + + def __call__(self, + **paras): + cfg = cfguh().cfg + cfgt = cfg.train + logm = distributed_log_manager() + epochn, itern, samplen = 0, 0, 0 + + step_type = cfgt.get('step_type', 'iter') + assert step_type in ['epoch', 'iter', 'sample'], \ + 'Step type must be in [epoch, iter, sample]' + + step_num = cfgt.get('step_num' , None) + gradacc_every = cfgt.get('gradacc_every', 1 ) + log_every = cfgt.get('log_every' , None) + ckpt_every = cfgt.get('ckpt_every' , None) + eval_start = cfgt.get('eval_start' , 0 ) + eval_every = cfgt.get('eval_every' , None) + + if paras.get('resume_step', None) is not None: + resume_step = paras['resume_step'] + assert step_type == resume_step['type'] + epochn = resume_step['epochn'] + itern = resume_step['itern'] + samplen = resume_step['samplen'] + del paras['resume_step'] + + trainloader = paras['trainloader'] + optimizer = paras['optimizer'] + scheduler = paras['scheduler'] + net = paras['net'] + + GRANK, LRANK, NRANK = sync.get_rank('all') + GWSIZE, LWSIZE, NODES = sync.get_world_size('all') + + weight_path = osp.join(cfgt.log_dir, 'weight') + if (GRANK==0) and (not osp.isdir(weight_path)): + os.makedirs(weight_path) + if (GRANK==0) and (cfgt.save_init_model): + self.save(net, is_init=True, step=0, optimizer=optimizer) + + epoch_time = timeit.default_timer() + end_flag = False + net.train() + + while True: + if step_type == 'epoch': + lr = scheduler[epochn] if scheduler is not None else None + for batch in trainloader: + # so first element of batch (usually image) can be [tensor] + if not isinstance(batch[0], list): + bs = batch[0].shape[0] + else: + bs = len(batch[0]) + if cfgt.skip_partial_batch and (bs != cfgt.batch_size_per_gpu): + continue + + itern_next = itern + 1 + samplen_next = samplen + bs*GWSIZE + + if step_type == 'iter': + lr = scheduler[itern//gradacc_every] if scheduler is not None else None + grad_update = itern%gradacc_every==(gradacc_every-1) + elif step_type == 'sample': + lr = scheduler[samplen] if scheduler is not None else None + # TODO: + # grad_update = samplen%gradacc_every==(gradacc_every-1) + + # timeDebug = timeit.default_timer() + paras_new = self.main( + batch=batch, + lr=lr, + itern=itern, + epochn=epochn, + samplen=samplen, + isinit=False, + grad_update=grad_update, + **paras) + # print_log(timeit.default_timer() - timeDebug) + + paras.update(paras_new) + logm.accumulate(bs, **paras['log_info']) + + ####### + # log # + ####### + + display_flag = False + if log_every is not None: + display_i = (itern//log_every) != (itern_next//log_every) + display_s = (samplen//log_every) != (samplen_next//log_every) + display_flag = (display_i and (step_type=='iter')) \ + or (display_s and (step_type=='sample')) + + if display_flag: + tbstep = itern_next if step_type=='iter' else samplen_next + console_info = logm.train_summary( + itern_next, epochn, samplen_next, lr, tbstep=tbstep) + logm.clear() + print_log(console_info) + + ######## + # eval # + ######## + + eval_flag = False + if (self.nested_eval_stage is not None) and (eval_every is not None) and (NRANK == 0): + if step_type=='iter': + eval_flag = (itern//eval_every) != (itern_next//eval_every) + eval_flag = eval_flag and (itern_next>=eval_start) + eval_flag = eval_flag or itern==0 + if step_type=='sample': + eval_flag = (samplen//eval_every) != (samplen_next//eval_every) + eval_flag = eval_flag and (samplen_next>=eval_start) + eval_flag = eval_flag or samplen==0 + + if eval_flag: + eval_cnt = itern_next if step_type=='iter' else samplen_next + net = self.set_model(net, 'eval') + rv = self.nested_eval_stage( + eval_cnt=eval_cnt, **paras) + rv = rv.get('eval_rv', None) + if rv is not None: + logm.tensorboard_log(eval_cnt, rv, mode='eval') + if self.is_better(rv): + self.rv_keep = rv + if GRANK==0: + step = {'epochn':epochn, 'itern':itern_next, + 'samplen':samplen_next, 'type':step_type, } + self.save(net, is_best=True, step=step, optimizer=optimizer) + net = self.set_model(net, 'train') + + ######## + # ckpt # + ######## + + ckpt_flag = False + if (GRANK==0) and (ckpt_every is not None): + # not distributed + ckpt_i = (itern//ckpt_every) != (itern_next//ckpt_every) + ckpt_s = (samplen//ckpt_every) != (samplen_next//ckpt_every) + ckpt_flag = (ckpt_i and (step_type=='iter')) \ + or (ckpt_s and (step_type=='sample')) + + if ckpt_flag: + if step_type == 'iter': + print_log('Checkpoint... {}'.format(itern_next)) + step = {'epochn':epochn, 'itern':itern_next, + 'samplen':samplen_next, 'type':step_type, } + self.save(net, itern=itern_next, step=step, optimizer=optimizer) + else: + print_log('Checkpoint... {}'.format(samplen_next)) + step = {'epochn':epochn, 'itern':itern_next, + 'samplen':samplen_next, 'type':step_type, } + self.save(net, samplen=samplen_next, step=step, optimizer=optimizer) + + ####### + # end # + ####### + + itern = itern_next + samplen = samplen_next + + if step_type is not None: + end_flag = (itern>=step_num and (step_type=='iter')) \ + or (samplen>=step_num and (step_type=='sample')) + if end_flag: + break + # loop end + + epochn += 1 + print_log('Epoch {} time:{:.2f}s.'.format( + epochn, timeit.default_timer()-epoch_time)) + epoch_time = timeit.default_timer() + + if end_flag: + break + elif step_type != 'epoch': + # This is temporarily added to resolve the data issue + trainloader = self.trick_update_trainloader(trainloader) + continue + + ####### + # log # + ####### + + display_flag = False + if (log_every is not None) and (step_type=='epoch'): + display_flag = (epochn==1) or (epochn%log_every==0) + + if display_flag: + console_info = logm.train_summary( + itern, epochn, samplen, lr, tbstep=epochn) + logm.clear() + print_log(console_info) + + ######## + # eval # + ######## + + eval_flag = False + if (self.nested_eval_stage is not None) and (eval_every is not None) \ + and (step_type=='epoch') and (NRANK==0): + eval_flag = (epochn%eval_every==0) and (itern_next>=eval_start) + eval_flag = (epochn==1) or eval_flag + + if eval_flag: + net = self.set_model(net, 'eval') + rv = self.nested_eval_stage( + eval_cnt=epochn, + **paras)['eval_rv'] + if rv is not None: + logm.tensorboard_log(epochn, rv, mode='eval') + if self.is_better(rv): + self.rv_keep = rv + if (GRANK==0): + step = {'epochn':epochn, 'itern':itern, + 'samplen':samplen, 'type':step_type, } + self.save(net, is_best=True, step=step, optimizer=optimizer) + net = self.set_model(net, 'train') + + ######## + # ckpt # + ######## + + ckpt_flag = False + if (ckpt_every is not None) and (GRANK==0) and (step_type=='epoch'): + # not distributed + ckpt_flag = epochn%ckpt_every==0 + + if ckpt_flag: + print_log('Checkpoint... {}'.format(itern_next)) + step = {'epochn':epochn, 'itern':itern, + 'samplen':samplen, 'type':step_type, } + self.save(net, epochn=epochn, step=step, optimizer=optimizer) + + ####### + # end # + ####### + if (step_type=='epoch') and (epochn>=step_num): + break + # loop end + + # This is temporarily added to resolve the data issue + trainloader = self.trick_update_trainloader(trainloader) + + logm.tensorboard_close() + return {} + + def main(self, **paras): + raise NotImplementedError + + def trick_update_trainloader(self, trainloader): + return trainloader + + def save_model(self, net, path_noext, **paras): + cfgt = cfguh().cfg.train + path = path_noext+'.pth' + if isinstance(net, (torch.nn.DataParallel, + torch.nn.parallel.DistributedDataParallel)): + netm = net.module + else: + netm = net + torch.save(netm.state_dict(), path) + print_log('Saving model file {0}'.format(path)) + + def save(self, net, itern=None, epochn=None, samplen=None, + is_init=False, is_best=False, is_last=False, **paras): + exid = cfguh().cfg.env.experiment_id + cfgt = cfguh().cfg.train + cfgm = cfguh().cfg.model + if isinstance(net, (torch.nn.DataParallel, + torch.nn.parallel.DistributedDataParallel)): + netm = net.module + else: + netm = net + net_symbol = cfgm.symbol + + check = sum([ + itern is not None, samplen is not None, epochn is not None, + is_init, is_best, is_last]) + assert check<2 + + if itern is not None: + path_noexp = '{}_{}_iter_{}'.format(exid, net_symbol, itern) + elif samplen is not None: + path_noexp = '{}_{}_samplen_{}'.format(exid, net_symbol, samplen) + elif epochn is not None: + path_noexp = '{}_{}_epoch_{}'.format(exid, net_symbol, epochn) + elif is_init: + path_noexp = '{}_{}_init'.format(exid, net_symbol) + elif is_best: + path_noexp = '{}_{}_best'.format(exid, net_symbol) + elif is_last: + path_noexp = '{}_{}_last'.format(exid, net_symbol) + else: + path_noexp = '{}_{}_default'.format(exid, net_symbol) + + path_noexp = osp.join(cfgt.log_dir, 'weight', path_noexp) + self.save_model(net, path_noexp, **paras) + +class eval_stage(object): + def __init__(self): + self.evaluator = None + + def create_dir(self, path): + local_rank = sync.get_rank('local') + if (not osp.isdir(path)) and (local_rank == 0): + os.makedirs(path) + sync.nodewise_sync().barrier() + + def __call__(self, + evalloader, + net, + **paras): + cfgt = cfguh().cfg.eval + local_rank = sync.get_rank('local') + if self.evaluator is None: + evaluator = get_evaluator()(cfgt.evaluator) + self.evaluator = evaluator + else: + evaluator = self.evaluator + + time_check = timeit.default_timer() + + for idx, batch in enumerate(evalloader): + rv = self.main(batch, net) + evaluator.add_batch(**rv) + if cfgt.output_result: + try: + self.output_f(**rv, cnt=paras['eval_cnt']) + except: + self.output_f(**rv) + if idx%cfgt.log_display == cfgt.log_display-1: + print_log('processed.. {}, Time:{:.2f}s'.format( + idx+1, timeit.default_timer() - time_check)) + time_check = timeit.default_timer() + # break + + evaluator.set_sample_n(len(evalloader.dataset)) + eval_rv = evaluator.compute() + if local_rank == 0: + evaluator.one_line_summary() + evaluator.save(cfgt.log_dir) + evaluator.clear_data() + return { + 'eval_rv' : eval_rv + } + +class exec_container(object): + """ + This is the base functor for all types of executions. + One execution can have multiple stages, + but are only allowed to use the same + config, network, dataloader. + Thus, in most of the cases, one exec_container is one + training/evaluation/demo... + If DPP is in use, this functor should be spawn. + """ + def __init__(self, + cfg, + **kwargs): + self.cfg = cfg + self.registered_stages = [] + self.node_rank = None + self.local_rank = None + self.global_rank = None + self.local_world_size = None + self.global_world_size = None + self.nodewise_sync_global_obj = sync.nodewise_sync_global() + + def register_stage(self, stage): + self.registered_stages.append(stage) + + def __call__(self, + local_rank, + **kwargs): + cfg = self.cfg + cfguh().save_cfg(cfg) + + self.node_rank = cfg.env.node_rank + self.local_rank = local_rank + self.nodes = cfg.env.nodes + self.local_world_size = cfg.env.gpu_count + + self.global_rank = self.local_rank + self.node_rank * self.nodes + self.global_world_size = self.nodes * self.local_world_size + + dist.init_process_group( + backend = cfg.env.dist_backend, + init_method = cfg.env.dist_url, + rank = self.global_rank, + world_size = self.global_world_size,) + torch.cuda.set_device(local_rank) + sync.nodewise_sync().copy_global(self.nodewise_sync_global_obj).local_init() + + if isinstance(cfg.env.rnd_seed, int): + np.random.seed(cfg.env.rnd_seed + self.global_rank) + torch.manual_seed(cfg.env.rnd_seed + self.global_rank) + + time_start = timeit.default_timer() + + para = {'itern_total' : 0,} + dl_para = self.prepare_dataloader() + assert isinstance(dl_para, dict) + para.update(dl_para) + + md_para = self.prepare_model() + assert isinstance(md_para, dict) + para.update(md_para) + + for stage in self.registered_stages: + stage_para = stage(**para) + if stage_para is not None: + para.update(stage_para) + + if self.global_rank==0: + self.save_last_model(**para) + + print_log( + 'Total {:.2f} seconds'.format(timeit.default_timer() - time_start)) + dist.destroy_process_group() + + def prepare_dataloader(self): + """ + Prepare the dataloader from config. + """ + return { + 'trainloader' : None, + 'evalloader' : None} + + def prepare_model(self): + """ + Prepare the model from config. + """ + return {'net' : None} + + def save_last_model(self, **para): + return + + def destroy(self): + self.nodewise_sync_global_obj.destroy() + +class train(exec_container): + def prepare_dataloader(self): + cfg = cfguh().cfg + trainset = get_dataset()(cfg.train.dataset) + sampler = get_sampler()( + dataset=trainset, cfg=cfg.train.dataset.get('sampler', 'default_train')) + trainloader = torch.utils.data.DataLoader( + trainset, + batch_size = cfg.train.batch_size_per_gpu, + sampler = sampler, + num_workers = cfg.train.dataset_num_workers_per_gpu, + drop_last = False, + pin_memory = cfg.train.dataset.get('pin_memory', False), + collate_fn = collate(),) + + evalloader = None + if 'eval' in cfg: + evalset = get_dataset()(cfg.eval.dataset) + if evalset is not None: + sampler = get_sampler()( + dataset=evalset, cfg=cfg.eval.dataset.get('sampler', 'default_eval')) + evalloader = torch.utils.data.DataLoader( + evalset, + batch_size = cfg.eval.batch_size_per_gpu, + sampler = sampler, + num_workers = cfg.eval.dataset_num_workers_per_gpu, + drop_last = False, + pin_memory = cfg.eval.dataset.get('pin_memory', False), + collate_fn = collate(),) + + return { + 'trainloader' : trainloader, + 'evalloader' : evalloader,} + + def prepare_model(self): + cfg = cfguh().cfg + net = get_model()(cfg.model) + if cfg.env.cuda: + net.to(self.local_rank) + net = torch.nn.parallel.DistributedDataParallel( + net, device_ids=[self.local_rank], + find_unused_parameters=True) + net.train() + scheduler = get_scheduler()(cfg.train.scheduler) + optimizer = get_optimizer()(net, cfg.train.optimizer) + return { + 'net' : net, + 'optimizer' : optimizer, + 'scheduler' : scheduler,} + + def save_last_model(self, **para): + cfgt = cfguh().cfg.train + net = para['net'] + net_symbol = cfguh().cfg.model.symbol + if isinstance(net, (torch.nn.DataParallel, + torch.nn.parallel.DistributedDataParallel)): + netm = net.module + else: + netm = net + path = osp.join(cfgt.log_dir, '{}_{}_last.pth'.format( + cfgt.experiment_id, net_symbol)) + torch.save(netm.state_dict(), path) + print_log('Saving model file {0}'.format(path)) + +class eval(exec_container): + def prepare_dataloader(self): + cfg = cfguh().cfg + evalloader = None + if cfg.eval.get('dataset', None) is not None: + evalset = get_dataset()(cfg.eval.dataset) + if evalset is None: + return + sampler = get_sampler()( + dataset=evalset, cfg=getattr(cfg.eval.dataset, 'sampler', 'default_eval')) + evalloader = torch.utils.data.DataLoader( + evalset, + batch_size = cfg.eval.batch_size_per_gpu, + sampler = sampler, + num_workers = cfg.eval.dataset_num_workers_per_gpu, + drop_last = False, + pin_memory = False, + collate_fn = collate(), ) + return { + 'trainloader' : None, + 'evalloader' : evalloader,} + + def prepare_model(self): + cfg = cfguh().cfg + net = get_model()(cfg.model) + if cfg.env.cuda: + net.to(self.local_rank) + net = torch.nn.parallel.DistributedDataParallel( + net, device_ids=[self.local_rank], + find_unused_parameters=True) + net.eval() + return {'net' : net,} + + def save_last_model(self, **para): + return + +############### +# some helper # +############### + +def torch_to_numpy(*argv): + if len(argv) > 1: + data = list(argv) + else: + data = argv[0] + + if isinstance(data, torch.Tensor): + return data.to('cpu').detach().numpy() + elif isinstance(data, (list, tuple)): + out = [] + for di in data: + out.append(torch_to_numpy(di)) + return out + elif isinstance(data, dict): + out = {} + for ni, di in data.items(): + out[ni] = torch_to_numpy(di) + return out + else: + return data + +import importlib + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit(".", 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) diff --git a/versatile_diffusion/main.py b/versatile_diffusion/main.py new file mode 100644 index 0000000000000000000000000000000000000000..efe56645b986c29c2ba0d85ee14b083d778f1ed0 --- /dev/null +++ b/versatile_diffusion/main.py @@ -0,0 +1,47 @@ +import torch.distributed as dist +import torch.multiprocessing as mp + +import os +import os.path as osp +import sys +import numpy as np +import copy + +from lib.cfg_holder import cfg_unique_holder as cfguh +from lib.cfg_helper import \ + get_command_line_args, \ + cfg_initiates + +from lib.model_zoo.sd import version +from lib.utils import get_obj_from_str + +if __name__ == "__main__": + cfg = get_command_line_args() + cfg = cfg_initiates(cfg) + + if 'train' in cfg: + trainer = get_obj_from_str(cfg.train.main)(cfg) + tstage = get_obj_from_str(cfg.train.stage)() + if 'eval' in cfg: + tstage.nested_eval_stage = get_obj_from_str(cfg.eval.stage)() + trainer.register_stage(tstage) + if cfg.env.gpu_count == 1: + trainer(0) + else: + mp.spawn(trainer, + args=(), + nprocs=cfg.env.gpu_count, + join=True) + trainer.destroy() + else: + evaler = get_obj_from_str(cfg.eval.main)(cfg) + estage = get_obj_from_str(cfg.eval.stage)() + evaler.register_stage(estage) + if cfg.env.gpu_count == 1: + evaler(0) + else: + mp.spawn(evaler, + args=(), + nprocs=cfg.env.gpu_count, + join=True) + evaler.destroy() diff --git a/versatile_diffusion/reconstruct_images.py b/versatile_diffusion/reconstruct_images.py new file mode 100644 index 0000000000000000000000000000000000000000..a0d4208e77697ba6d32776e3e6b795248a555053 --- /dev/null +++ b/versatile_diffusion/reconstruct_images.py @@ -0,0 +1,126 @@ +import os +import os.path as osp +import PIL +from PIL import Image +from pathlib import Path +import numpy as np +import numpy.random as npr + +import torch +import torchvision.transforms as tvtrans +from lib.cfg_helper import model_cfg_bank +from lib.model_zoo import get_model +from lib.model_zoo.ddim_vd import DDIMSampler_VD +from lib.experiments.sd_default import color_adjust, auto_merge_imlist +from torch.utils.data import DataLoader, Dataset +import argparse + +from lib.model_zoo.vd import VD +from lib.cfg_holder import cfg_unique_holder as cfguh +from lib.cfg_helper import get_command_line_args, cfg_initiates, load_cfg_yaml +import matplotlib.pyplot as plt + +from skimage.transform import resize, downscale_local_mean + +def regularize_image(x): + BICUBIC = PIL.Image.Resampling.BICUBIC + if isinstance(x, str): + x = Image.open(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, PIL.Image.Image): + x = x.resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, np.ndarray): + x = PIL.Image.fromarray(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, torch.Tensor): + pass + else: + assert False, 'Unknown image type' + + assert (x.shape[1]==512) & (x.shape[2]==512), \ + 'Wrong image size' + return x + +cfgm_name = 'vd_noema' +sampler = DDIMSampler_VD +pth = '/home/furkan/Versatile-Diffusion/pretrained/vd-four-flow-v1-0-fp16.pth' +cfgm = model_cfg_bank()(cfgm_name) +net = get_model()(cfgm) +sd = torch.load(pth, map_location='cpu') +net.load_state_dict(sd, strict=False) + +#net.model.cuda(1) +sampler = sampler(net) +#sampler.model.model.cuda(1) +#sampler.model.cuda(1) +net.clip.cuda(0) +net.autokl.cuda(0).half() +sampler.model.model.diffusion_model.device='cuda:1' +sampler.model.model.diffusion_model.half().cuda(1) + +pred_clip = np.load('/home/furkan/Versatile-Diffusion/extractedfeatures/nsd/nsd_clipvision_predtest_sepembeds.npy') +pred_clip = torch.tensor(pred_clip).half().cuda(1) + +n_samples = 1 +ddim_steps = 50 +ddim_eta = 0 +scale = 7.5 +xtype = 'image' +ctype = 'vision' +h, w = 512,512 +shape = [n_samples, 4, h//8, w//8] + +u = None +if scale != 1.0: + dummy = torch.zeros((1,3,224,224)).cuda(0) + u = net.clip_encode_vision(dummy) + u = u.cuda(1).half() + +torch.manual_seed(0) +idx = [35,70,13,40,54,78,97,102,451,461,570,774] +for i in idx: #range(len(pred_clip)): + + c = pred_clip[i:i+1] + + + z, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=c, + unconditional_guidance_scale=scale, + unconditional_conditioning=u, + xtype=xtype, ctype=ctype, + eta=ddim_eta, + verbose=False,) + + + z = z.cuda(0) + x = net.autokl_decode(z) + #x = torch.clamp((x+1.0)/2.0, min=0.0, max=1.0) + #x = [tvtrans.ToPILImage()(xi) for xi in x] + + im = Image.open('/home/furkan/NSD/nsddata_stimuli/test_images/test_image{}.png'.format(i)) + im = regularize_image(im) + cin = im*2 - 1 + color_adj='None' + color_adj_to = cin + color_adj_flag = (color_adj!='none') and (color_adj!='None') and (color_adj is not None) + color_adj_simple = (color_adj=='Simple') or color_adj=='simple' + color_adj_keep_ratio = 0.5 + + if color_adj_flag and (ctype=='vision'): + x_adj = [] + for xi in x: + color_adj_f = color_adjust(ref_from=(xi+1)/2, ref_to=color_adj_to) + xi_adj = color_adj_f((xi+1)/2, keep=color_adj_keep_ratio, simple=color_adj_simple) + x_adj.append(xi_adj) + x = x_adj + x = [tvtrans.ToPILImage()(xi) for xi in x] + else: + x = torch.clamp((x+1.0)/2.0, min=0.0, max=1.0) + x = [tvtrans.ToPILImage()(xi) for xi in x] + + x[0].save('/home/furkan/Versatile-Diffusion/results/nsd_clipvision_nocolorcalibration/{}.png'.format(i)) + + diff --git a/versatile_diffusion/reconstruct_txt2im.py b/versatile_diffusion/reconstruct_txt2im.py new file mode 100644 index 0000000000000000000000000000000000000000..b04d848d14427b2c2060291aa819e340afd0fa41 --- /dev/null +++ b/versatile_diffusion/reconstruct_txt2im.py @@ -0,0 +1,126 @@ +import os +import os.path as osp +import PIL +from PIL import Image +from pathlib import Path +import numpy as np +import numpy.random as npr + +import torch +import torchvision.transforms as tvtrans +from lib.cfg_helper import model_cfg_bank +from lib.model_zoo import get_model +from lib.model_zoo.ddim_vd import DDIMSampler_VD +from lib.experiments.sd_default import color_adjust, auto_merge_imlist +from torch.utils.data import DataLoader, Dataset +import argparse + +from lib.model_zoo.vd import VD +from lib.cfg_holder import cfg_unique_holder as cfguh +from lib.cfg_helper import get_command_line_args, cfg_initiates, load_cfg_yaml +import matplotlib.pyplot as plt + +from skimage.transform import resize, downscale_local_mean + +def regularize_image(x): + BICUBIC = PIL.Image.Resampling.BICUBIC + if isinstance(x, str): + x = Image.open(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, PIL.Image.Image): + x = x.resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, np.ndarray): + x = PIL.Image.fromarray(x).resize([512, 512], resample=BICUBIC) + x = tvtrans.ToTensor()(x) + elif isinstance(x, torch.Tensor): + pass + else: + assert False, 'Unknown image type' + + assert (x.shape[1]==512) & (x.shape[2]==512), \ + 'Wrong image size' + return x + +cfgm_name = 'vd_noema' +sampler = DDIMSampler_VD +pth = '/home/furkan/Versatile-Diffusion/pretrained/vd-four-flow-v1-0-fp16.pth' +cfgm = model_cfg_bank()(cfgm_name) +net = get_model()(cfgm) +sd = torch.load(pth, map_location='cpu') +net.load_state_dict(sd, strict=False) + +#net.model.cuda(1) +sampler = sampler(net) +#sampler.model.model.cuda(1) +#sampler.model.cuda(1) +net.clip.cuda(0) +net.autokl.cuda(0).half() +sampler.model.model.diffusion_model.device='cuda:1' +sampler.model.model.diffusion_model.half().cuda(1) + +pred_clip = np.load('/home/furkan/Versatile-Diffusion/extractedfeatures/nsd/nsd_cliptext_predtest_sepembeds.npy') +pred_clip = torch.tensor(pred_clip).half().cuda(1) + +n_samples = 1 +ddim_steps = 50 +ddim_eta = 0 +scale = 7.5 +xtype = 'image' +ctype = 'prompt' +h, w = 512,512 +shape = [n_samples, 4, h//8, w//8] + +u = None +if scale != 1.0: + dummy = '' + u = net.clip_encode_text(dummy) + u = u.cuda(1).half() + +torch.manual_seed(0) +idx = [35,70,13,40,54,78,97,102,451,461,570,774] +for i in idx: #range(len(pred_clip)): + + c = pred_clip[i:i+1] + + + z, _ = sampler.sample( + steps=ddim_steps, + shape=shape, + conditioning=c, + unconditional_guidance_scale=scale, + unconditional_conditioning=u, + xtype=xtype, ctype=ctype, + eta=ddim_eta, + verbose=False,) + + + z = z.cuda(0) + x = net.autokl_decode(z) + #x = torch.clamp((x+1.0)/2.0, min=0.0, max=1.0) + #x = [tvtrans.ToPILImage()(xi) for xi in x] + + im = Image.open('/home/furkan/NSD/nsddata_stimuli/test_images/test_image{}.png'.format(i)) + im = regularize_image(im) + cin = im*2 - 1 + color_adj='None' + color_adj_to = cin + color_adj_flag = (color_adj!='none') and (color_adj!='None') and (color_adj is not None) + color_adj_simple = (color_adj=='Simple') or color_adj=='simple' + color_adj_keep_ratio = 0.5 + + if color_adj_flag and (ctype=='vision'): + x_adj = [] + for xi in x: + color_adj_f = color_adjust(ref_from=(xi+1)/2, ref_to=color_adj_to) + xi_adj = color_adj_f((xi+1)/2, keep=color_adj_keep_ratio, simple=color_adj_simple) + x_adj.append(xi_adj) + x = x_adj + x = [tvtrans.ToPILImage()(xi) for xi in x] + else: + x = torch.clamp((x+1.0)/2.0, min=0.0, max=1.0) + x = [tvtrans.ToPILImage()(xi) for xi in x] + + x[0].save('/home/furkan/Versatile-Diffusion/results/nsd_trials/nsd_cliptext_fmri/{}.png'.format(i)) + + diff --git a/versatile_diffusion/requirement.txt b/versatile_diffusion/requirement.txt new file mode 100644 index 0000000000000000000000000000000000000000..3759835801b6509b14d5a9dd0c02726b5e31ca1e --- /dev/null +++ b/versatile_diffusion/requirement.txt @@ -0,0 +1,16 @@ +matplotlib==3.4.2 +pyyaml==5.4.1 +opencv-python==4.5.1.48 +easydict==1.9 +scikit-image==0.17.2 +tensorboardx==2.1 +tensorboard==1.15.0 +lpips==0.1.3 + +tqdm==4.60.0 +transformers==4.24.0 +torchmetrics==0.7.3 + +einops==0.3.0 +omegaconf==2.1.1 +-e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers diff --git a/versatile_diffusion/requirement_colab.txt b/versatile_diffusion/requirement_colab.txt new file mode 100644 index 0000000000000000000000000000000000000000..c1f5f63ba3422c21b884d830f8a6a291c5d523c9 --- /dev/null +++ b/versatile_diffusion/requirement_colab.txt @@ -0,0 +1,13 @@ +matplotlib==3.4.2 +pyyaml==5.4.1 +opencv-python==4.5.1.48 +easydict==1.9 +scikit-image==0.17.2 +lpips==0.1.3 + +tqdm==4.60.0 +transformers==4.24.0 +torchmetrics==0.7.3 + +einops==0.3.0 +omegaconf==2.1.1