Upload 4 files
Browse files
CNN.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5a829c1267d78d4f8a5276de0fdaef87335ab07fdfce6addd4784618a1edcec9
|
| 3 |
+
size 4070660
|
CNN.py
CHANGED
|
@@ -1,31 +1,21 @@
|
|
| 1 |
"""
|
| 2 |
-
|
| 3 |
-
==================================
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
-> light CNN boundary refinement
|
| 14 |
-
|
| 15 |
-
This treats watershed-like basins as over-segmentation proposals, not labels.
|
| 16 |
-
The learnable part decides which candidates are physical wave systems, which
|
| 17 |
-
should be merged, and which should be sent to the downstream self-pruning VAE.
|
| 18 |
-
|
| 19 |
-
v4.5 is designed for a double-layer physical teacher: peak-core proposals define
|
| 20 |
-
system identity/count, while valley-constrained support proposals recover the full
|
| 21 |
-
energetic wave-system footprint. Stripe-like bands can be attached as tails but
|
| 22 |
-
are not allowed to become independent systems without a peak core.
|
| 23 |
-
|
| 24 |
-
All operations are lightweight and Colab-friendly. No Transformer blocks.
|
| 25 |
"""
|
|
|
|
|
|
|
| 26 |
import math
|
| 27 |
from dataclasses import dataclass, asdict
|
| 28 |
-
from typing import Optional, Dict
|
| 29 |
|
| 30 |
import torch
|
| 31 |
import torch.nn as nn
|
|
@@ -33,27 +23,28 @@ import torch.nn.functional as F
|
|
| 33 |
|
| 34 |
|
| 35 |
@dataclass
|
| 36 |
-
class
|
| 37 |
n_freqs: int = 47
|
| 38 |
n_dirs: int = 72
|
| 39 |
n_slots: int = 6
|
| 40 |
-
|
| 41 |
-
|
| 42 |
prop_feat_dim: int = 22
|
| 43 |
width: int = 32
|
| 44 |
depth: int = 4
|
| 45 |
-
node_dim: int =
|
| 46 |
-
|
|
|
|
| 47 |
pair_feat_dim: int = 8
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
|
| 58 |
def to_dict(self):
|
| 59 |
return asdict(self)
|
|
@@ -74,8 +65,8 @@ class DepthwiseSeparable(nn.Module):
|
|
| 74 |
class PhysicsAwareModule(nn.Module):
|
| 75 |
def __init__(self, ch: int, n_dirs: int):
|
| 76 |
super().__init__()
|
| 77 |
-
self.n_dirs = n_dirs
|
| 78 |
dirs = torch.linspace(0, 2 * math.pi, n_dirs + 1)[:n_dirs]
|
|
|
|
| 79 |
self.register_buffer("cos_d", torch.cos(dirs).view(1, 1, 1, n_dirs))
|
| 80 |
self.register_buffer("sin_d", torch.sin(dirs).view(1, 1, 1, n_dirs))
|
| 81 |
self.fuse = nn.Conv2d(ch + 3, ch, 1, bias=False)
|
|
@@ -97,7 +88,7 @@ class PhysicsAwareModule(nn.Module):
|
|
| 97 |
dir_conc = dir_conc.expand(-1, -1, -1, self.n_dirs)
|
| 98 |
fcoord = torch.linspace(0, 1, nf, device=E.device, dtype=E.dtype).view(1, 1, nf, 1)
|
| 99 |
fc = (En * fcoord).sum(dim=2, keepdim=True) / col_sum
|
| 100 |
-
spread = torch.sqrt((
|
| 101 |
spread = spread.expand(-1, -1, nf, -1)
|
| 102 |
return torch.cat([hf_tail, dir_conc, spread], dim=1)
|
| 103 |
|
|
@@ -105,244 +96,360 @@ class PhysicsAwareModule(nn.Module):
|
|
| 105 |
return self.act(self.norm(self.fuse(torch.cat([h, self._phys_features(E)], dim=1))))
|
| 106 |
|
| 107 |
|
| 108 |
-
class
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
super().__init__()
|
| 123 |
-
self.cfg = cfg or
|
| 124 |
c = self.cfg
|
| 125 |
-
|
| 126 |
-
|
|
|
|
| 127 |
self.stem_norm = nn.GroupNorm(min(8, c.width), c.width)
|
| 128 |
-
self.physics = PhysicsAwareModule(c.width, c.n_dirs)
|
| 129 |
self.blocks = nn.ModuleList([DepthwiseSeparable(c.width, c.width) for _ in range(c.depth)])
|
| 130 |
self.global_pool = nn.AdaptiveAvgPool2d(1)
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
self.node_mlp = nn.Sequential(
|
| 136 |
nn.Linear(c.width + c.prop_feat_dim, c.node_dim), nn.GELU(),
|
| 137 |
nn.Linear(c.node_dim, c.node_dim), nn.GELU(),
|
| 138 |
)
|
| 139 |
self.node_keep = nn.Linear(c.node_dim, 1)
|
| 140 |
-
self.
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
)
|
| 144 |
-
# Edge head uses node_i, node_j, absolute difference, product, plus handcrafted physical pair features.
|
| 145 |
self.edge_head = nn.Sequential(
|
| 146 |
-
nn.Linear(4 * c.node_dim + c.pair_feat_dim, c.
|
| 147 |
-
nn.Linear(c.
|
| 148 |
)
|
| 149 |
-
self._coord_cache = None
|
| 150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
|
|
|
|
|
|
|
|
|
| 155 |
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
|
| 163 |
def _coord_channels(self, B, device, dtype):
|
| 164 |
-
c = self.cfg
|
| 165 |
if self._coord_cache is None:
|
| 166 |
-
nf, nd =
|
| 167 |
fcoord = torch.linspace(0, 1, nf).view(1, 1, nf, 1).expand(1, 1, nf, nd)
|
| 168 |
ang = torch.linspace(0, 2 * math.pi, nd + 1)[:nd].view(1, 1, 1, nd).expand(1, 1, nf, nd)
|
| 169 |
self._coord_cache = torch.cat([fcoord, torch.sin(ang), torch.cos(ang)], dim=1)
|
| 170 |
return self._coord_cache.to(device=device, dtype=dtype).expand(B, -1, -1, -1)
|
| 171 |
|
| 172 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
c = self.cfg
|
| 174 |
-
B,
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
d_f = (fi - fj).abs()
|
| 200 |
-
si =
|
| 201 |
-
sj =
|
| 202 |
-
dot = (si * sj + ci * cj).clamp(-1
|
| 203 |
d_t = torch.acos(dot) / math.pi
|
| 204 |
-
mi =
|
| 205 |
-
mj = prop_feats[:, :, 0][:, None, :]
|
| 206 |
d_m = (mi - mj).abs()
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
d_sf = (
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
c = self.cfg
|
| 227 |
B, _, H, W = x.shape
|
| 228 |
-
prop_masks = torch.nan_to_num(prop_masks.float(), nan=0.0
|
| 229 |
-
prop_feats = torch.nan_to_num(prop_feats.float(), nan=0.0, posinf=5.0, neginf=-5.0).clamp(-5
|
| 230 |
prop_valid = prop_valid.float().clamp(0, 1)
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
for blk in self.blocks:
|
| 237 |
-
h =
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
assign = node_slot * node_keep[:, :, None]
|
| 256 |
-
prior_signal = torch.einsum("bpk,bphw->bkhw", assign,
|
| 257 |
-
# Normalize each slot prior but preserve zero slots.
|
| 258 |
prior_signal = prior_signal / prior_signal.amax(dim=(2, 3), keepdim=True).clamp_min(1e-6)
|
| 259 |
-
prior_signal =
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
return {
|
| 277 |
-
"
|
| 278 |
-
"
|
| 279 |
-
"
|
| 280 |
-
"
|
| 281 |
-
"
|
| 282 |
-
"
|
| 283 |
-
"count_logits": count_logits,
|
| 284 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
}
|
| 286 |
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
WaveSystemGraphParserV48 = WaveSystemGraphParserV47
|
|
|
|
| 1 |
"""
|
| 2 |
+
WaveSystem Set Parser V5 for ICWDS
|
| 3 |
+
==================================
|
| 4 |
+
A lightweight physics-seeded set parser for wave directional spectra.
|
| 5 |
+
|
| 6 |
+
Key changes from v4.8:
|
| 7 |
+
- dual proposal source: external physics proposals + learned seed proposals;
|
| 8 |
+
- multi-round relation message passing;
|
| 9 |
+
- slot set prediction with per-slot existence probabilities;
|
| 10 |
+
- no count-soft -> round -> top-N hard deletion;
|
| 11 |
+
- separate core and support mask heads;
|
| 12 |
+
- auxiliary wind-sea and wave-age heads for real weak supervision.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
import math
|
| 17 |
from dataclasses import dataclass, asdict
|
| 18 |
+
from typing import Optional, Dict, Tuple
|
| 19 |
|
| 20 |
import torch
|
| 21 |
import torch.nn as nn
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
@dataclass
|
| 26 |
+
class WaveSystemSetParserV5Config:
|
| 27 |
n_freqs: int = 47
|
| 28 |
n_dirs: int = 72
|
| 29 |
n_slots: int = 6
|
| 30 |
+
p_phys_max: int = 24
|
| 31 |
+
learned_seed_k: int = 8
|
| 32 |
prop_feat_dim: int = 22
|
| 33 |
width: int = 32
|
| 34 |
depth: int = 4
|
| 35 |
+
node_dim: int = 64
|
| 36 |
+
slot_dim: int = 64
|
| 37 |
+
msg_rounds: int = 3
|
| 38 |
pair_feat_dim: int = 8
|
| 39 |
+
seed_sigma_f: float = 0.085
|
| 40 |
+
seed_sigma_t: float = 0.16 # fraction of pi
|
| 41 |
+
prior_gain_core: float = 3.2
|
| 42 |
+
prior_gain_support: float = 2.0
|
| 43 |
+
min_exist_bias: float = -0.8
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def p_total(self) -> int:
|
| 47 |
+
return self.p_phys_max + self.learned_seed_k
|
| 48 |
|
| 49 |
def to_dict(self):
|
| 50 |
return asdict(self)
|
|
|
|
| 65 |
class PhysicsAwareModule(nn.Module):
|
| 66 |
def __init__(self, ch: int, n_dirs: int):
|
| 67 |
super().__init__()
|
|
|
|
| 68 |
dirs = torch.linspace(0, 2 * math.pi, n_dirs + 1)[:n_dirs]
|
| 69 |
+
self.n_dirs = n_dirs
|
| 70 |
self.register_buffer("cos_d", torch.cos(dirs).view(1, 1, 1, n_dirs))
|
| 71 |
self.register_buffer("sin_d", torch.sin(dirs).view(1, 1, 1, n_dirs))
|
| 72 |
self.fuse = nn.Conv2d(ch + 3, ch, 1, bias=False)
|
|
|
|
| 88 |
dir_conc = dir_conc.expand(-1, -1, -1, self.n_dirs)
|
| 89 |
fcoord = torch.linspace(0, 1, nf, device=E.device, dtype=E.dtype).view(1, 1, nf, 1)
|
| 90 |
fc = (En * fcoord).sum(dim=2, keepdim=True) / col_sum
|
| 91 |
+
spread = torch.sqrt((En * (fcoord - fc) ** 2).sum(dim=2, keepdim=True) / col_sum)
|
| 92 |
spread = spread.expand(-1, -1, nf, -1)
|
| 93 |
return torch.cat([hf_tail, dir_conc, spread], dim=1)
|
| 94 |
|
|
|
|
| 96 |
return self.act(self.norm(self.fuse(torch.cat([h, self._phys_features(E)], dim=1))))
|
| 97 |
|
| 98 |
|
| 99 |
+
class RelationBlock(nn.Module):
|
| 100 |
+
def __init__(self, dim: int, pair_dim: int):
|
| 101 |
+
super().__init__()
|
| 102 |
+
self.q = nn.Linear(dim, dim, bias=False)
|
| 103 |
+
self.k = nn.Linear(dim, dim, bias=False)
|
| 104 |
+
self.v = nn.Linear(dim, dim, bias=False)
|
| 105 |
+
self.edge_bias = nn.Sequential(
|
| 106 |
+
nn.Linear(pair_dim, dim // 2), nn.GELU(), nn.Linear(dim // 2, 1)
|
| 107 |
+
)
|
| 108 |
+
self.update = nn.Sequential(
|
| 109 |
+
nn.Linear(dim * 2, dim * 2), nn.GELU(), nn.Linear(dim * 2, dim)
|
| 110 |
+
)
|
| 111 |
+
self.norm = nn.LayerNorm(dim)
|
| 112 |
+
|
| 113 |
+
def forward(self, node, pair_feat, valid):
|
| 114 |
+
# node [B,P,D], pair_feat [B,P,P,F], valid [B,P]
|
| 115 |
+
D = node.shape[-1]
|
| 116 |
+
q = self.q(node)
|
| 117 |
+
k = self.k(node)
|
| 118 |
+
v = self.v(node)
|
| 119 |
+
logits = torch.einsum("bpd,bqd->bpq", q, k) / math.sqrt(D)
|
| 120 |
+
logits = logits + self.edge_bias(pair_feat).squeeze(-1)
|
| 121 |
+
pair_valid = valid[:, :, None].bool() & valid[:, None, :].bool()
|
| 122 |
+
logits = logits.masked_fill(~pair_valid, -30.0)
|
| 123 |
+
attn = torch.softmax(logits, dim=-1)
|
| 124 |
+
attn = torch.nan_to_num(attn, nan=0.0)
|
| 125 |
+
msg = torch.einsum("bpq,bqd->bpd", attn, v)
|
| 126 |
+
upd = self.update(torch.cat([node, msg], dim=-1))
|
| 127 |
+
out = self.norm(node + upd)
|
| 128 |
+
return out * valid[:, :, None]
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class WaveSystemSetParserV5(nn.Module):
|
| 132 |
+
def __init__(self, cfg: Optional[WaveSystemSetParserV5Config] = None):
|
| 133 |
super().__init__()
|
| 134 |
+
self.cfg = cfg or WaveSystemSetParserV5Config()
|
| 135 |
c = self.cfg
|
| 136 |
+
|
| 137 |
+
# Names intentionally match the v4.8 backbone for partial warm-start.
|
| 138 |
+
self.stem = nn.Conv2d(4, c.width, 3, padding=1)
|
| 139 |
self.stem_norm = nn.GroupNorm(min(8, c.width), c.width)
|
| 140 |
+
self.physics = PhysicsAwareModule(c.width, c.n_dirs)
|
| 141 |
self.blocks = nn.ModuleList([DepthwiseSeparable(c.width, c.width) for _ in range(c.depth)])
|
| 142 |
self.global_pool = nn.AdaptiveAvgPool2d(1)
|
| 143 |
+
|
| 144 |
+
self.seed_head = nn.Sequential(
|
| 145 |
+
nn.Conv2d(c.width, c.width, 3, padding=1), nn.GELU(), nn.Conv2d(c.width, 1, 1)
|
| 146 |
+
)
|
| 147 |
+
self.global_support_head = nn.Sequential(
|
| 148 |
+
nn.Conv2d(c.width, c.width, 3, padding=1), nn.GELU(), nn.Conv2d(c.width, 1, 1)
|
| 149 |
+
)
|
| 150 |
|
| 151 |
self.node_mlp = nn.Sequential(
|
| 152 |
nn.Linear(c.width + c.prop_feat_dim, c.node_dim), nn.GELU(),
|
| 153 |
nn.Linear(c.node_dim, c.node_dim), nn.GELU(),
|
| 154 |
)
|
| 155 |
self.node_keep = nn.Linear(c.node_dim, 1)
|
| 156 |
+
self.relation = nn.ModuleList([
|
| 157 |
+
RelationBlock(c.node_dim, c.pair_feat_dim) for _ in range(c.msg_rounds)
|
| 158 |
+
])
|
|
|
|
|
|
|
| 159 |
self.edge_head = nn.Sequential(
|
| 160 |
+
nn.Linear(4 * c.node_dim + c.pair_feat_dim, c.node_dim), nn.GELU(),
|
| 161 |
+
nn.Linear(c.node_dim, c.node_dim), nn.GELU(), nn.Linear(c.node_dim, 1)
|
| 162 |
)
|
|
|
|
| 163 |
|
| 164 |
+
self.slot_queries = nn.Parameter(torch.randn(c.n_slots, c.slot_dim) * 0.02)
|
| 165 |
+
self.slot_q = nn.Linear(c.slot_dim, c.node_dim, bias=False)
|
| 166 |
+
self.slot_k = nn.Linear(c.node_dim, c.node_dim, bias=False)
|
| 167 |
+
self.slot_v = nn.Linear(c.node_dim, c.slot_dim, bias=False)
|
| 168 |
+
self.slot_update = nn.Sequential(
|
| 169 |
+
nn.Linear(c.slot_dim * 2, c.slot_dim * 2), nn.GELU(), nn.Linear(c.slot_dim * 2, c.slot_dim)
|
| 170 |
+
)
|
| 171 |
+
self.slot_norm = nn.LayerNorm(c.slot_dim)
|
| 172 |
+
self.exist_head = nn.Linear(c.slot_dim, 1)
|
| 173 |
+
nn.init.constant_(self.exist_head.bias, c.min_exist_bias)
|
| 174 |
+
self.count_head = nn.Sequential(
|
| 175 |
+
nn.Linear(c.width + c.slot_dim, c.slot_dim), nn.GELU(), nn.Linear(c.slot_dim, c.n_slots + 1)
|
| 176 |
+
)
|
| 177 |
|
| 178 |
+
self.pixel_embed = nn.Conv2d(c.width, c.slot_dim, 1, bias=False)
|
| 179 |
+
self.core_slot_proj = nn.Linear(c.slot_dim, c.slot_dim, bias=False)
|
| 180 |
+
self.support_slot_proj = nn.Linear(c.slot_dim, c.slot_dim, bias=False)
|
| 181 |
+
self.bg_head = nn.Sequential(
|
| 182 |
+
nn.Conv2d(c.width, c.width, 3, padding=1), nn.GELU(), nn.Conv2d(c.width, 1, 1)
|
| 183 |
+
)
|
| 184 |
|
| 185 |
+
self.windsea_head = nn.Sequential(
|
| 186 |
+
nn.Conv2d(c.width, c.width // 2, 3, padding=1), nn.GELU(), nn.Conv2d(c.width // 2, 1, 1)
|
| 187 |
+
)
|
| 188 |
+
self.wave_age_head = nn.Sequential(
|
| 189 |
+
nn.Conv2d(c.width, c.width // 2, 3, padding=1), nn.GELU(), nn.Conv2d(c.width // 2, 1, 1)
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
self.register_buffer("fgrid", torch.linspace(0, 1, c.n_freqs).view(1, 1, c.n_freqs, 1))
|
| 193 |
+
ang = torch.linspace(0, 2 * math.pi, c.n_dirs + 1)[:c.n_dirs]
|
| 194 |
+
self.register_buffer("tgrid", ang.view(1, 1, 1, c.n_dirs))
|
| 195 |
+
self.register_buffer("sin_grid", torch.sin(ang).view(1, 1, 1, c.n_dirs))
|
| 196 |
+
self.register_buffer("cos_grid", torch.cos(ang).view(1, 1, 1, c.n_dirs))
|
| 197 |
+
self._coord_cache = None
|
| 198 |
|
| 199 |
def _coord_channels(self, B, device, dtype):
|
|
|
|
| 200 |
if self._coord_cache is None:
|
| 201 |
+
nf, nd = self.cfg.n_freqs, self.cfg.n_dirs
|
| 202 |
fcoord = torch.linspace(0, 1, nf).view(1, 1, nf, 1).expand(1, 1, nf, nd)
|
| 203 |
ang = torch.linspace(0, 2 * math.pi, nd + 1)[:nd].view(1, 1, 1, nd).expand(1, 1, nf, nd)
|
| 204 |
self._coord_cache = torch.cat([fcoord, torch.sin(ang), torch.cos(ang)], dim=1)
|
| 205 |
return self._coord_cache.to(device=device, dtype=dtype).expand(B, -1, -1, -1)
|
| 206 |
|
| 207 |
+
@staticmethod
|
| 208 |
+
def _safe_softmax(x, dim=-1):
|
| 209 |
+
x = torch.nan_to_num(x, nan=0.0, posinf=30.0, neginf=-30.0).clamp(-30, 30)
|
| 210 |
+
x = x - x.max(dim=dim, keepdim=True).values.detach()
|
| 211 |
+
return torch.nan_to_num(torch.softmax(x, dim=dim), nan=0.0)
|
| 212 |
+
|
| 213 |
+
def _periodic_max_pool(self, x, kernel=3):
|
| 214 |
+
pad = kernel // 2
|
| 215 |
+
xp = torch.cat([x[..., -pad:], x, x[..., :pad]], dim=-1)
|
| 216 |
+
yp = F.max_pool2d(xp, kernel_size=kernel, stride=1, padding=(pad, 0))
|
| 217 |
+
return yp
|
| 218 |
+
|
| 219 |
+
def _learned_proposals(self, E01, seed_logits, support_logits) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 220 |
c = self.cfg
|
| 221 |
+
B, _, H, W = seed_logits.shape
|
| 222 |
+
seed = torch.sigmoid(seed_logits)
|
| 223 |
+
support = torch.sigmoid(support_logits)
|
| 224 |
+
pooled = self._periodic_max_pool(seed, 3)
|
| 225 |
+
peaks = seed * (seed >= pooled - 1e-6).float()
|
| 226 |
+
vals, idx = torch.topk(peaks.flatten(1), k=c.learned_seed_k, dim=1)
|
| 227 |
+
rr = (idx // W).float() / max(H - 1, 1)
|
| 228 |
+
cc = idx % W
|
| 229 |
+
theta = cc.float() / W * (2 * math.pi)
|
| 230 |
+
|
| 231 |
+
fg = self.fgrid.to(E01.dtype)
|
| 232 |
+
tg = self.tgrid.to(E01.dtype)
|
| 233 |
+
df = (fg - rr[:, :, None, None]) / c.seed_sigma_f
|
| 234 |
+
dt = torch.atan2(
|
| 235 |
+
torch.sin(tg - theta[:, :, None, None]),
|
| 236 |
+
torch.cos(tg - theta[:, :, None, None]),
|
| 237 |
+
) / (math.pi * c.seed_sigma_t)
|
| 238 |
+
gauss = torch.exp(-0.5 * (df * df + dt * dt))
|
| 239 |
+
masks = (gauss * (0.15 + 0.85 * support)).clamp(0, 1)
|
| 240 |
+
valid = (vals > 0.03).float()
|
| 241 |
+
masks = masks * valid[:, :, None, None]
|
| 242 |
+
|
| 243 |
+
total = E01.sum(dim=(2, 3)).clamp_min(1e-6)
|
| 244 |
+
mass = torch.einsum("bchw,bphw->bp", E01, masks) / total
|
| 245 |
+
peak = vals.clamp(0, 1)
|
| 246 |
+
area = masks.mean(dim=(2, 3))
|
| 247 |
+
sf = torch.full_like(mass, c.seed_sigma_f)
|
| 248 |
+
st = torch.full_like(mass, c.seed_sigma_t)
|
| 249 |
+
zeros = torch.zeros_like(mass)
|
| 250 |
+
ones = torch.ones_like(mass)
|
| 251 |
+
sin_t = torch.sin(theta)
|
| 252 |
+
cos_t = torch.cos(theta)
|
| 253 |
+
quality = peak
|
| 254 |
+
feats = torch.stack([
|
| 255 |
+
mass, peak, area, rr, sin_t, cos_t, sf, st,
|
| 256 |
+
area, rr, torch.full_like(mass, c.learned_seed_k / float(c.p_total)), quality,
|
| 257 |
+
peak, zeros, ones * 0.5, mass, ones, ones, zeros, peak, ones, zeros,
|
| 258 |
+
], dim=-1)
|
| 259 |
+
return masks, feats, valid
|
| 260 |
+
|
| 261 |
+
def _node_pool(self, h, masks, valid):
|
| 262 |
+
denom = masks.flatten(2).sum(dim=2).clamp_min(1.0)
|
| 263 |
+
pooled = torch.einsum("bchw,bphw->bpc", h, masks) / denom[:, :, None]
|
| 264 |
+
return pooled * valid[:, :, None]
|
| 265 |
+
|
| 266 |
+
def _pair_features(self, feats):
|
| 267 |
+
fi = feats[:, :, 3][:, :, None]; fj = feats[:, :, 3][:, None, :]
|
| 268 |
d_f = (fi - fj).abs()
|
| 269 |
+
si = feats[:, :, 4][:, :, None]; ci = feats[:, :, 5][:, :, None]
|
| 270 |
+
sj = feats[:, :, 4][:, None, :]; cj = feats[:, :, 5][:, None, :]
|
| 271 |
+
dot = (si * sj + ci * cj).clamp(-1 + 1e-5, 1 - 1e-5)
|
| 272 |
d_t = torch.acos(dot) / math.pi
|
| 273 |
+
mi = feats[:, :, 0][:, :, None]; mj = feats[:, :, 0][:, None, :]
|
|
|
|
| 274 |
d_m = (mi - mj).abs()
|
| 275 |
+
sfi = feats[:, :, 6][:, :, None]; sfj = feats[:, :, 6][:, None, :]
|
| 276 |
+
sti = feats[:, :, 7][:, :, None]; stj = feats[:, :, 7][:, None, :]
|
| 277 |
+
d_sf = (sfi - sfj).abs(); d_st = (sti - stj).abs()
|
| 278 |
+
prom = torch.minimum(feats[:, :, 12][:, :, None], feats[:, :, 12][:, None, :])
|
| 279 |
+
stripe = torch.maximum(feats[:, :, 13][:, :, None], feats[:, :, 13][:, None, :])
|
| 280 |
+
qual = torch.minimum(feats[:, :, 11][:, :, None], feats[:, :, 11][:, None, :])
|
| 281 |
+
return torch.stack([d_f, d_t, d_m, d_sf, d_st, prom, stripe, qual], dim=-1)
|
| 282 |
+
|
| 283 |
+
def _slot_reasoning(self, node, node_keep, valid):
|
| 284 |
+
B = node.shape[0]
|
| 285 |
+
slots = self.slot_queries[None].expand(B, -1, -1)
|
| 286 |
+
key = self.slot_k(node)
|
| 287 |
+
value = self.slot_v(node)
|
| 288 |
+
for _ in range(2):
|
| 289 |
+
q = self.slot_q(slots)
|
| 290 |
+
score = torch.einsum("bkd,bpd->bkp", q, key) / math.sqrt(key.shape[-1])
|
| 291 |
+
score = score + torch.log(node_keep[:, None, :].clamp_min(1e-4))
|
| 292 |
+
score = score.masked_fill(valid[:, None, :] <= 0, -30.0)
|
| 293 |
+
attn = self._safe_softmax(score, dim=-1)
|
| 294 |
+
ctx = torch.einsum("bkp,bpd->bkd", attn, value)
|
| 295 |
+
slots = self.slot_norm(slots + self.slot_update(torch.cat([slots, ctx], dim=-1)))
|
| 296 |
+
return slots
|
| 297 |
+
|
| 298 |
+
def forward(self, x, prop_masks, prop_feats, prop_valid) -> Dict[str, torch.Tensor]:
|
| 299 |
c = self.cfg
|
| 300 |
B, _, H, W = x.shape
|
| 301 |
+
prop_masks = torch.nan_to_num(prop_masks.float(), nan=0.0).clamp(0, 1)
|
| 302 |
+
prop_feats = torch.nan_to_num(prop_feats.float(), nan=0.0, posinf=5.0, neginf=-5.0).clamp(-5, 5)
|
| 303 |
prop_valid = prop_valid.float().clamp(0, 1)
|
| 304 |
+
|
| 305 |
+
h_in = torch.cat([x, self._coord_channels(B, x.device, x.dtype)], dim=1)
|
| 306 |
+
h = F.gelu(self.stem_norm(self.stem(h_in)))
|
| 307 |
+
E01 = ((x + 1.0) * 0.5).clamp(0, 1)
|
| 308 |
+
h = h + self.physics(h, E01)
|
| 309 |
for blk in self.blocks:
|
| 310 |
+
h = h + blk(h)
|
| 311 |
+
|
| 312 |
+
seed_logits = self.seed_head(h)
|
| 313 |
+
global_support_logits = self.global_support_head(h)
|
| 314 |
+
learned_masks, learned_feats, learned_valid = self._learned_proposals(E01, seed_logits, global_support_logits)
|
| 315 |
+
|
| 316 |
+
masks = torch.cat([prop_masks, learned_masks], dim=1)
|
| 317 |
+
feats = torch.cat([prop_feats, learned_feats], dim=1)
|
| 318 |
+
valid = torch.cat([prop_valid, learned_valid], dim=1)
|
| 319 |
+
|
| 320 |
+
node_pool = self._node_pool(h, masks, valid)
|
| 321 |
+
node = self.node_mlp(torch.cat([node_pool, feats], dim=-1)) * valid[:, :, None]
|
| 322 |
+
pair_feat = self._pair_features(feats)
|
| 323 |
+
for block in self.relation:
|
| 324 |
+
node = block(node, pair_feat, valid)
|
| 325 |
+
|
| 326 |
+
node_keep_logit = self.node_keep(node).squeeze(-1).masked_fill(valid <= 0, -20.0)
|
| 327 |
+
node_keep = torch.sigmoid(node_keep_logit) * valid
|
| 328 |
+
|
| 329 |
+
# Pair merge probability is diagnostic/training supervision; set prediction itself is not hard-merged.
|
| 330 |
+
P = node.shape[1]
|
| 331 |
+
ni = node[:, :, None, :].expand(B, P, P, -1)
|
| 332 |
+
nj = node[:, None, :, :].expand(B, P, P, -1)
|
| 333 |
+
edge_in = torch.cat([ni, nj, (ni - nj).abs(), ni * nj, pair_feat], dim=-1)
|
| 334 |
+
edge_logits = self.edge_head(edge_in).squeeze(-1)
|
| 335 |
+
pair_valid = valid[:, :, None].bool() & valid[:, None, :].bool()
|
| 336 |
+
eye = torch.eye(P, device=x.device, dtype=torch.bool)[None]
|
| 337 |
+
edge_valid = pair_valid & (~eye)
|
| 338 |
+
edge_logits = edge_logits.masked_fill(~edge_valid, 0.0)
|
| 339 |
+
|
| 340 |
+
slots = self._slot_reasoning(node, node_keep, valid)
|
| 341 |
+
exist_logit = self.exist_head(slots).squeeze(-1)
|
| 342 |
+
exist_prob = torch.sigmoid(exist_logit)
|
| 343 |
+
|
| 344 |
+
slot_key = self.slot_q(slots)
|
| 345 |
+
node_slot_logits = torch.einsum("bpd,bkd->bpk", node, slot_key) / math.sqrt(node.shape[-1])
|
| 346 |
+
node_slot_logits = node_slot_logits.masked_fill(valid[:, :, None] <= 0, -20.0)
|
| 347 |
+
node_slot = self._safe_softmax(node_slot_logits, dim=-1) * valid[:, :, None]
|
| 348 |
assign = node_slot * node_keep[:, :, None]
|
| 349 |
+
prior_signal = torch.einsum("bpk,bphw->bkhw", assign, masks)
|
|
|
|
| 350 |
prior_signal = prior_signal / prior_signal.amax(dim=(2, 3), keepdim=True).clamp_min(1e-6)
|
| 351 |
+
prior_signal = prior_signal.clamp(0, 1)
|
| 352 |
+
prior_expand = F.max_pool2d(prior_signal, 5, stride=1, padding=2)
|
| 353 |
+
|
| 354 |
+
pix = self.pixel_embed(h)
|
| 355 |
+
core_vec = self.core_slot_proj(slots)
|
| 356 |
+
support_vec = self.support_slot_proj(slots)
|
| 357 |
+
core_logits = torch.einsum("bkd,bdhw->bkhw", core_vec, pix) / math.sqrt(c.slot_dim)
|
| 358 |
+
support_logits = torch.einsum("bkd,bdhw->bkhw", support_vec, pix) / math.sqrt(c.slot_dim)
|
| 359 |
+
core_logits = core_logits + c.prior_gain_core * (prior_signal - 0.35)
|
| 360 |
+
support_logits = support_logits + c.prior_gain_support * (prior_expand - 0.30) + 1.25 * (torch.sigmoid(global_support_logits) - 0.5)
|
| 361 |
+
core_prob = torch.sigmoid(core_logits)
|
| 362 |
+
support_prob = torch.sigmoid(support_logits)
|
| 363 |
+
mask_prob = (support_prob * (0.35 + 0.65 * core_prob)).clamp(0, 1)
|
| 364 |
+
|
| 365 |
+
slot_score = mask_prob * exist_prob[:, :, None, None]
|
| 366 |
+
bg_logit = self.bg_head(h)
|
| 367 |
+
bg_score = torch.sigmoid(bg_logit)
|
| 368 |
+
scores = torch.cat([slot_score, bg_score], dim=1).clamp_min(1e-6)
|
| 369 |
+
prob = scores / scores.sum(dim=1, keepdim=True).clamp_min(1e-6)
|
| 370 |
+
|
| 371 |
+
global_feat = self.global_pool(h).flatten(1)
|
| 372 |
+
count_logits = self.count_head(torch.cat([global_feat, slots.mean(dim=1)], dim=-1))
|
| 373 |
+
|
| 374 |
return {
|
| 375 |
+
"prob": prob,
|
| 376 |
+
"mask_prob": mask_prob,
|
| 377 |
+
"core_prob": core_prob,
|
| 378 |
+
"support_prob": support_prob,
|
| 379 |
+
"exist_logit": exist_logit,
|
| 380 |
+
"exist_prob": exist_prob,
|
| 381 |
+
"count_logits": count_logits,
|
| 382 |
+
"seed_logits": seed_logits,
|
| 383 |
+
"global_support_logits": global_support_logits,
|
| 384 |
+
"windsea_logits": self.windsea_head(h),
|
| 385 |
+
"wave_age_pred": torch.sigmoid(self.wave_age_head(h)),
|
| 386 |
+
"node_keep_logit": node_keep_logit,
|
| 387 |
+
"node_slot_logits": node_slot_logits,
|
| 388 |
+
"edge_logits": edge_logits,
|
| 389 |
+
"edge_valid": edge_valid,
|
| 390 |
+
"all_prop_masks": masks,
|
| 391 |
+
"all_prop_valid": valid,
|
| 392 |
+
"prior_signal": prior_signal,
|
| 393 |
}
|
| 394 |
|
| 395 |
+
|
| 396 |
+
def transfer_v48_backbone(model: WaveSystemSetParserV5, state_dict: Dict[str, torch.Tensor]) -> Dict[str, float]:
|
| 397 |
+
"""Load shape-compatible v4.8 backbone tensors into V5."""
|
| 398 |
+
target = model.state_dict()
|
| 399 |
+
prefixes = ("stem.", "stem_norm.", "physics.", "blocks.")
|
| 400 |
+
matched = {}
|
| 401 |
+
for k, v in state_dict.items():
|
| 402 |
+
kk = k
|
| 403 |
+
for pfx in ("module.", "model.", "cnn.", "seg_model.", "seg."):
|
| 404 |
+
if kk.startswith(pfx):
|
| 405 |
+
kk = kk[len(pfx):]
|
| 406 |
+
if kk.startswith(prefixes) and kk in target and tuple(v.shape) == tuple(target[kk].shape):
|
| 407 |
+
matched[kk] = v
|
| 408 |
+
model.load_state_dict(matched, strict=False)
|
| 409 |
+
total_backbone = sum(v.numel() for k, v in target.items() if k.startswith(prefixes))
|
| 410 |
+
loaded = sum(target[k].numel() for k in matched)
|
| 411 |
+
return {
|
| 412 |
+
"matched_tensors": len(matched),
|
| 413 |
+
"backbone_numel_coverage": loaded / max(total_backbone, 1),
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
# -----------------------------------------------------------------------------
|
| 417 |
+
# Archive/runtime helpers
|
| 418 |
+
# -----------------------------------------------------------------------------
|
| 419 |
+
def build_model(config: Optional[WaveSystemSetParserV5Config] = None) -> WaveSystemSetParserV5:
|
| 420 |
+
"""Construct the exact V5 segmentation architecture used by CNN.pt."""
|
| 421 |
+
return WaveSystemSetParserV5(config or WaveSystemSetParserV5Config())
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def load_cnn_checkpoint(
|
| 425 |
+
checkpoint_path: str,
|
| 426 |
+
map_location: str | torch.device = "cpu",
|
| 427 |
+
strict: bool = True,
|
| 428 |
+
) -> WaveSystemSetParserV5:
|
| 429 |
+
"""
|
| 430 |
+
Load the archived CNN.pt checkpoint.
|
| 431 |
+
|
| 432 |
+
CNN.pt is expected to be the renamed V5 phase-C complete checkpoint:
|
| 433 |
+
WaveSystemSetParserV5/checkpoints/phase_C/complete.pt
|
| 434 |
+
"""
|
| 435 |
+
ck = torch.load(checkpoint_path, map_location=map_location, weights_only=False)
|
| 436 |
+
cfg_dict = ck.get("config", {}) if isinstance(ck, dict) else {}
|
| 437 |
+
allowed = set(WaveSystemSetParserV5Config.__dataclass_fields__)
|
| 438 |
+
cfg = WaveSystemSetParserV5Config(**{k: v for k, v in cfg_dict.items() if k in allowed})
|
| 439 |
+
model = WaveSystemSetParserV5(cfg)
|
| 440 |
+
|
| 441 |
+
if isinstance(ck, dict):
|
| 442 |
+
state = ck.get("model", ck.get("state_dict", ck))
|
| 443 |
+
else:
|
| 444 |
+
state = ck
|
| 445 |
+
model.load_state_dict(state, strict=strict)
|
| 446 |
+
return model
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
__all__ = [
|
| 450 |
+
"WaveSystemSetParserV5Config",
|
| 451 |
+
"WaveSystemSetParserV5",
|
| 452 |
+
"transfer_v48_backbone",
|
| 453 |
+
"build_model",
|
| 454 |
+
"load_cnn_checkpoint",
|
| 455 |
+
]
|
|
|
VAE.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc6973ce13befa835c19120863678e65aadec7179f8a4f48a2949fe14a9efd62
|
| 3 |
+
size 34975280
|
VAE.py
CHANGED
|
@@ -1,31 +1,41 @@
|
|
| 1 |
"""
|
| 2 |
-
VAE
|
| 3 |
-
======
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
"""
|
|
|
|
|
|
|
| 26 |
import math
|
| 27 |
from dataclasses import dataclass, asdict
|
| 28 |
-
from typing import Optional, Tuple
|
| 29 |
|
| 30 |
import torch
|
| 31 |
import torch.nn as nn
|
|
@@ -33,188 +43,1264 @@ import torch.nn.functional as F
|
|
| 33 |
|
| 34 |
|
| 35 |
@dataclass
|
| 36 |
-
class
|
| 37 |
n_freqs: int = 47
|
| 38 |
n_dirs: int = 72
|
| 39 |
-
pad_freqs: int = 48
|
| 40 |
n_slots: int = 6
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
|
| 47 |
def to_dict(self):
|
| 48 |
return asdict(self)
|
| 49 |
|
| 50 |
|
| 51 |
-
class
|
| 52 |
-
def __init__(self,
|
| 53 |
super().__init__()
|
| 54 |
-
self.
|
| 55 |
-
|
| 56 |
-
self.
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
self.
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
nn.
|
| 69 |
-
nn.
|
| 70 |
-
nn.
|
| 71 |
-
nn.GroupNorm(min(8, w), w), nn.GELU(),
|
| 72 |
-
nn.Conv2d(w, w * 4, 3, padding=1), nn.PixelShuffle(2),
|
| 73 |
-
nn.GroupNorm(min(8, w), w), nn.GELU(),
|
| 74 |
-
nn.Conv2d(w, 1, 1),
|
| 75 |
)
|
| 76 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
return mu, logvar.clamp(-8.0, 5.0)
|
| 82 |
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
return mu
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
super().__init__()
|
| 98 |
-
self.cfg = cfg
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
self.
|
| 102 |
-
self.register_buffer(
|
| 103 |
-
self.desc_head = nn.Sequential(
|
| 104 |
-
nn.Linear(cfg.latent_dim, 64), nn.GELU(),
|
| 105 |
-
nn.Linear(64, 64), nn.GELU(),
|
| 106 |
-
nn.Linear(64, cfg.desc_dim),
|
| 107 |
-
)
|
| 108 |
-
|
| 109 |
-
def _pad(self, x: torch.Tensor):
|
| 110 |
-
if self.cfg.pad_freqs > self.cfg.n_freqs:
|
| 111 |
-
return F.pad(x, (0, 0, 0, self.cfg.pad_freqs - self.cfg.n_freqs))
|
| 112 |
-
return x
|
| 113 |
-
|
| 114 |
-
def gates(self, temperature: float = 1.0, hard: bool = False):
|
| 115 |
-
g = torch.sigmoid(self.gate_logits / max(float(temperature), 1e-4))
|
| 116 |
-
if hard:
|
| 117 |
-
gh = (g > self.cfg.gate_hard_th).float()
|
| 118 |
-
g = gh.detach() - g.detach() + g
|
| 119 |
-
return g
|
| 120 |
-
|
| 121 |
-
def ordered_mask(self, B: int, K: int, keep_min: int, keep_max: int, device):
|
| 122 |
-
D = self.cfg.latent_dim
|
| 123 |
-
lo = int(max(1, min(D, round(keep_min))))
|
| 124 |
-
hi = int(max(lo, min(D, round(keep_max))))
|
| 125 |
-
keep = torch.randint(lo, hi + 1, (B, K, 1), device=device)
|
| 126 |
-
dim = torch.arange(D, device=device).view(1, 1, D)
|
| 127 |
-
return (dim < keep).float()
|
| 128 |
|
| 129 |
def forward(
|
| 130 |
self,
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
):
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
z = self.vae.reparameterize(mu, logvar)
|
| 142 |
-
mu = mu.view(B, K, D)
|
| 143 |
-
logvar = logvar.view(B, K, D)
|
| 144 |
-
z = z.view(B, K, D)
|
| 145 |
-
|
| 146 |
-
gate = self.gates(gate_temperature, hard=hard_gate)[:K, :].unsqueeze(0)
|
| 147 |
-
z_used = z * gate
|
| 148 |
-
if self.training and ordered_keep is not None:
|
| 149 |
-
z_used = z_used * self.ordered_mask(B, K, ordered_keep[0], ordered_keep[1], parts.device)
|
| 150 |
-
if presence is not None:
|
| 151 |
-
z_used = z_used * presence.float().clamp(0, 1).unsqueeze(-1)
|
| 152 |
-
|
| 153 |
-
recon = self.vae.decode(z_used.reshape(B * K, D)).view(B, K, nf, nd)
|
| 154 |
-
desc = self.desc_head(z_used.reshape(B * K, D)).view(B, K, self.cfg.desc_dim)
|
| 155 |
-
return {'recon': recon, 'mu': mu, 'logvar': logvar, 'z': z, 'z_used': z_used, 'gate': gate.squeeze(0), 'desc': desc}
|
| 156 |
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
return {
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
'gate_hard_per_slot': hard.sum(dim=1).detach().cpu().numpy(),
|
| 167 |
-
'gate_per_slot': g.detach().cpu().numpy(),
|
| 168 |
}
|
| 169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
def num_params(self):
|
| 171 |
return sum(p.numel() for p in self.parameters())
|
| 172 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
+
V5-Aware Analytic-Centered Structured Semantic VAE (AC-SSVAE)
|
| 3 |
+
================================================================
|
| 4 |
+
Stage-2 feature extractor for ICWDS.
|
| 5 |
+
|
| 6 |
+
Design goals
|
| 7 |
+
------------
|
| 8 |
+
1. Consume WaveSystemSetParserV5 soft wave-system outputs instead of hard labels.
|
| 9 |
+
2. Use an analytic physical descriptor as the prior center, and learn only bounded
|
| 10 |
+
semantic corrections.
|
| 11 |
+
3. Canonicalize each wave system before shape encoding so the free latent does not
|
| 12 |
+
waste capacity on location, direction, scale, or energy.
|
| 13 |
+
4. Keep a small causal shape latent. The residual decoder is constructed as
|
| 14 |
+
Delta(z, s) = F(z, s) - F(0, s)
|
| 15 |
+
so z=0 exactly returns the semantic base.
|
| 16 |
+
5. Expose deterministic decoding APIs for quantization, intervention, and later
|
| 17 |
+
30-byte packet design.
|
| 18 |
+
|
| 19 |
+
Internal semantic vector (9D)
|
| 20 |
+
-----------------------------
|
| 21 |
+
[log_energy,
|
| 22 |
+
peak_frequency_01,
|
| 23 |
+
sin_peak_direction,
|
| 24 |
+
cos_peak_direction,
|
| 25 |
+
frequency_spread_01,
|
| 26 |
+
direction_spread_over_pi,
|
| 27 |
+
f_theta_correlation,
|
| 28 |
+
frequency_skew,
|
| 29 |
+
direction_skew]
|
| 30 |
+
|
| 31 |
+
The packet-facing direction is still one circular quantity. sin/cos are only the
|
| 32 |
+
neural internal representation.
|
| 33 |
"""
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
import math
|
| 37 |
from dataclasses import dataclass, asdict
|
| 38 |
+
from typing import Dict, Optional, Tuple
|
| 39 |
|
| 40 |
import torch
|
| 41 |
import torch.nn as nn
|
|
|
|
| 43 |
|
| 44 |
|
| 45 |
@dataclass
|
| 46 |
+
class V5AwareACSSVAEConfig:
|
| 47 |
n_freqs: int = 47
|
| 48 |
n_dirs: int = 72
|
|
|
|
| 49 |
n_slots: int = 6
|
| 50 |
+
semantic_dim: int = 9
|
| 51 |
+
shape_dim: int = 4
|
| 52 |
+
context_dim: int = 24
|
| 53 |
+
|
| 54 |
+
canonical_h: int = 32
|
| 55 |
+
canonical_w: int = 48
|
| 56 |
+
canonical_f_extent: float = 3.0
|
| 57 |
+
canonical_t_extent: float = 3.0
|
| 58 |
+
|
| 59 |
+
encoder_width: int = 48
|
| 60 |
+
hidden_dim: int = 160
|
| 61 |
+
decoder_width: int = 64
|
| 62 |
+
|
| 63 |
+
# Bounded semantic correction magnitudes around analytic center.
|
| 64 |
+
# [logE, f, sin/cos correction, spreads, rho, skews]
|
| 65 |
+
delta_loge: float = 0.35
|
| 66 |
+
delta_f: float = 0.08
|
| 67 |
+
delta_angle_rad: float = math.radians(18.0)
|
| 68 |
+
delta_log_spread: float = 0.45
|
| 69 |
+
delta_rho_logit: float = 0.75
|
| 70 |
+
delta_skew: float = 0.90
|
| 71 |
+
|
| 72 |
+
min_spread_f: float = 0.010
|
| 73 |
+
max_spread_f: float = 0.45
|
| 74 |
+
min_spread_t: float = 0.015
|
| 75 |
+
max_spread_t: float = 0.95
|
| 76 |
+
max_abs_rho: float = 0.92
|
| 77 |
+
max_abs_skew: float = 3.0
|
| 78 |
+
|
| 79 |
+
shape_log_residual_scale: float = 2.25
|
| 80 |
+
energy_log_den: float = math.log1p(47 * 72)
|
| 81 |
+
eps: float = 1e-6
|
| 82 |
|
| 83 |
def to_dict(self):
|
| 84 |
return asdict(self)
|
| 85 |
|
| 86 |
|
| 87 |
+
class ConvNormAct(nn.Module):
|
| 88 |
+
def __init__(self, ci: int, co: int, stride: int = 1):
|
| 89 |
super().__init__()
|
| 90 |
+
self.conv = nn.Conv2d(ci, co, 3, stride=stride, padding=1, bias=False)
|
| 91 |
+
self.norm = nn.GroupNorm(min(8, co), co)
|
| 92 |
+
self.act = nn.GELU()
|
| 93 |
+
|
| 94 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 95 |
+
return self.act(self.norm(self.conv(x)))
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class ResBlock(nn.Module):
|
| 99 |
+
def __init__(self, ch: int):
|
| 100 |
+
super().__init__()
|
| 101 |
+
self.net = nn.Sequential(
|
| 102 |
+
nn.Conv2d(ch, ch, 3, padding=1, bias=False),
|
| 103 |
+
nn.GroupNorm(min(8, ch), ch),
|
| 104 |
+
nn.GELU(),
|
| 105 |
+
nn.Conv2d(ch, ch, 3, padding=1, bias=False),
|
| 106 |
+
nn.GroupNorm(min(8, ch), ch),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
)
|
| 108 |
+
self.act = nn.GELU()
|
| 109 |
+
|
| 110 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 111 |
+
return self.act(x + self.net(x))
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _wrap_angle(x: torch.Tensor) -> torch.Tensor:
|
| 115 |
+
return torch.atan2(torch.sin(x), torch.cos(x))
|
| 116 |
+
|
| 117 |
|
| 118 |
+
def _atanh_safe(x: torch.Tensor, eps: float = 1e-5) -> torch.Tensor:
|
| 119 |
+
x = x.clamp(-1 + eps, 1 - eps)
|
| 120 |
+
return 0.5 * (torch.log1p(x) - torch.log1p(-x))
|
|
|
|
| 121 |
|
| 122 |
+
|
| 123 |
+
def reparameterize(mu: torch.Tensor, logvar: torch.Tensor, stochastic: bool) -> torch.Tensor:
|
| 124 |
+
if not stochastic:
|
| 125 |
return mu
|
| 126 |
+
std = torch.exp(0.5 * logvar)
|
| 127 |
+
return mu + std * torch.randn_like(std)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
@torch.no_grad()
|
| 131 |
+
def analytic_semantics(
|
| 132 |
+
energy: torch.Tensor,
|
| 133 |
+
mask_prob: torch.Tensor,
|
| 134 |
+
core_prob: Optional[torch.Tensor] = None,
|
| 135 |
+
eps: float = 1e-6,
|
| 136 |
+
energy_log_den: Optional[float] = None,
|
| 137 |
+
) -> torch.Tensor:
|
| 138 |
+
"""Compute stable 9D physical descriptors from soft wave-system masks.
|
| 139 |
+
|
| 140 |
+
Parameters
|
| 141 |
+
----------
|
| 142 |
+
energy: [B,1,H,W] or [B,H,W], non-negative display-domain energy.
|
| 143 |
+
mask_prob: [B,K,H,W].
|
| 144 |
+
core_prob: optional [B,K,H,W]. Core weighting sharpens peak estimation.
|
| 145 |
+
"""
|
| 146 |
+
if energy.ndim == 3:
|
| 147 |
+
energy = energy[:, None]
|
| 148 |
+
B, K, H, W = mask_prob.shape
|
| 149 |
+
dtype, device = energy.dtype, energy.device
|
| 150 |
+
if energy_log_den is None:
|
| 151 |
+
energy_log_den = math.log1p(H * W)
|
| 152 |
+
|
| 153 |
+
E = torch.nan_to_num(energy, nan=0.0, posinf=1.0, neginf=0.0).clamp_min(0)
|
| 154 |
+
M = torch.nan_to_num(mask_prob, nan=0.0).clamp(0, 1)
|
| 155 |
+
part = E * M
|
| 156 |
+
mass = part.sum(dim=(-2, -1)).clamp_min(eps)
|
| 157 |
+
loge = torch.log1p(mass) / float(energy_log_den)
|
| 158 |
+
|
| 159 |
+
fgrid = torch.linspace(0, 1, H, device=device, dtype=dtype).view(1, 1, H, 1)
|
| 160 |
+
theta = torch.linspace(0, 2 * math.pi, W + 1, device=device, dtype=dtype)[:W]
|
| 161 |
+
tgrid = theta.view(1, 1, 1, W)
|
| 162 |
|
| 163 |
+
peak_src = part
|
| 164 |
+
if core_prob is not None:
|
| 165 |
+
core = torch.nan_to_num(core_prob, nan=0.0).clamp(0, 1)
|
| 166 |
+
peak_src = part * (0.25 + 0.75 * core)
|
| 167 |
|
| 168 |
+
# Smooth soft-peak estimator. It is more stable than hard argmax on noisy spectra.
|
| 169 |
+
pnorm = peak_src / peak_src.amax(dim=(-2, -1), keepdim=True).clamp_min(eps)
|
| 170 |
+
peak_w = torch.softmax((12.0 * pnorm).flatten(2), dim=-1).view(B, K, H, W)
|
| 171 |
+
fp = (peak_w * fgrid).sum(dim=(-2, -1))
|
| 172 |
+
sx = (peak_w * torch.sin(tgrid)).sum(dim=(-2, -1))
|
| 173 |
+
cx = (peak_w * torch.cos(tgrid)).sum(dim=(-2, -1))
|
| 174 |
+
theta_p = torch.atan2(sx, cx)
|
| 175 |
+
sinp = torch.sin(theta_p)
|
| 176 |
+
cosp = torch.cos(theta_p)
|
| 177 |
|
| 178 |
+
w = part / mass[:, :, None, None]
|
| 179 |
+
df = fgrid - fp[:, :, None, None]
|
| 180 |
+
dt = _wrap_angle(tgrid - theta_p[:, :, None, None]) / math.pi
|
| 181 |
+
|
| 182 |
+
sf = torch.sqrt((w * df.square()).sum(dim=(-2, -1)).clamp_min(eps))
|
| 183 |
+
st = torch.sqrt((w * dt.square()).sum(dim=(-2, -1)).clamp_min(eps))
|
| 184 |
+
|
| 185 |
+
zf = df / sf[:, :, None, None].clamp_min(1e-3)
|
| 186 |
+
zt = dt / st[:, :, None, None].clamp_min(1e-3)
|
| 187 |
+
rho = (w * zf * zt).sum(dim=(-2, -1)).clamp(-0.95, 0.95)
|
| 188 |
+
skew_f = (w * zf.pow(3)).sum(dim=(-2, -1)).clamp(-4.0, 4.0)
|
| 189 |
+
skew_t = (w * zt.pow(3)).sum(dim=(-2, -1)).clamp(-4.0, 4.0)
|
| 190 |
+
|
| 191 |
+
return torch.stack([
|
| 192 |
+
loge, fp, sinp, cosp, sf, st, rho, skew_f, skew_t
|
| 193 |
+
], dim=-1)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class Canonicalizer(nn.Module):
|
| 197 |
+
"""Differentiably center/scale each slot in frequency-direction coordinates."""
|
| 198 |
+
def __init__(self, cfg: V5AwareACSSVAEConfig):
|
| 199 |
super().__init__()
|
| 200 |
+
self.cfg = cfg
|
| 201 |
+
u = torch.linspace(-cfg.canonical_f_extent, cfg.canonical_f_extent, cfg.canonical_h)
|
| 202 |
+
v = torch.linspace(-cfg.canonical_t_extent, cfg.canonical_t_extent, cfg.canonical_w)
|
| 203 |
+
self.register_buffer("u", u.view(1, 1, cfg.canonical_h, 1))
|
| 204 |
+
self.register_buffer("v", v.view(1, 1, 1, cfg.canonical_w))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
|
| 206 |
def forward(
|
| 207 |
self,
|
| 208 |
+
energy: torch.Tensor,
|
| 209 |
+
mask_prob: torch.Tensor,
|
| 210 |
+
core_prob: torch.Tensor,
|
| 211 |
+
support_prob: torch.Tensor,
|
| 212 |
+
semantics: torch.Tensor,
|
| 213 |
+
) -> torch.Tensor:
|
| 214 |
+
if energy.ndim == 3:
|
| 215 |
+
energy = energy[:, None]
|
| 216 |
+
B, K, H, W = mask_prob.shape
|
| 217 |
+
eps = self.cfg.eps
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
|
| 219 |
+
E = energy[:, None].expand(B, K, 1, H, W)
|
| 220 |
+
M = mask_prob[:, :, None]
|
| 221 |
+
C = core_prob[:, :, None]
|
| 222 |
+
S = support_prob[:, :, None]
|
| 223 |
+
part = E * M
|
| 224 |
+
cpart = E * C
|
| 225 |
+
spart = E * S
|
| 226 |
+
|
| 227 |
+
peak = part.amax(dim=(-2, -1), keepdim=True).clamp_min(eps)
|
| 228 |
+
channels = torch.cat([
|
| 229 |
+
part / peak,
|
| 230 |
+
cpart / peak,
|
| 231 |
+
spart / peak,
|
| 232 |
+
M,
|
| 233 |
+
C,
|
| 234 |
+
S,
|
| 235 |
+
], dim=2).reshape(B * K, 6, H, W)
|
| 236 |
+
|
| 237 |
+
sem = semantics.reshape(B * K, -1)
|
| 238 |
+
fp = sem[:, 1].clamp(0, 1)
|
| 239 |
+
theta = torch.atan2(sem[:, 2], sem[:, 3]) % (2 * math.pi)
|
| 240 |
+
sf = sem[:, 4].clamp(self.cfg.min_spread_f, self.cfg.max_spread_f)
|
| 241 |
+
st = sem[:, 5].clamp(self.cfg.min_spread_t, self.cfg.max_spread_t)
|
| 242 |
+
|
| 243 |
+
uf = self.u.to(channels.dtype)
|
| 244 |
+
vt = self.v.to(channels.dtype)
|
| 245 |
+
f_sample = fp[:, None, None, None] + sf[:, None, None, None] * uf
|
| 246 |
+
theta_sample = theta[:, None, None, None] + (math.pi * st[:, None, None, None]) * vt
|
| 247 |
+
theta_frac = torch.remainder(theta_sample, 2 * math.pi) / (2 * math.pi)
|
| 248 |
+
|
| 249 |
+
# Tile direction three times and sample from the middle copy.
|
| 250 |
+
tiled = torch.cat([channels, channels, channels], dim=-1)
|
| 251 |
+
y = 2.0 * f_sample - 1.0
|
| 252 |
+
xpix = (1.0 + theta_frac) * W - 0.5
|
| 253 |
+
x = 2.0 * xpix / max(3 * W - 1, 1) - 1.0
|
| 254 |
+
grid = torch.stack([
|
| 255 |
+
x.expand(-1, -1, self.cfg.canonical_h, self.cfg.canonical_w).squeeze(1),
|
| 256 |
+
y.expand(-1, -1, self.cfg.canonical_h, self.cfg.canonical_w).squeeze(1),
|
| 257 |
+
], dim=-1)
|
| 258 |
+
out = F.grid_sample(
|
| 259 |
+
tiled,
|
| 260 |
+
grid,
|
| 261 |
+
mode="bilinear",
|
| 262 |
+
padding_mode="zeros",
|
| 263 |
+
align_corners=True,
|
| 264 |
+
)
|
| 265 |
+
return out.reshape(B, K, 6, self.cfg.canonical_h, self.cfg.canonical_w)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
class SlotEncoder(nn.Module):
|
| 269 |
+
def __init__(self, cfg: V5AwareACSSVAEConfig):
|
| 270 |
+
super().__init__()
|
| 271 |
+
w = cfg.encoder_width
|
| 272 |
+
self.image_net = nn.Sequential(
|
| 273 |
+
ConvNormAct(6, w, 1),
|
| 274 |
+
ResBlock(w),
|
| 275 |
+
ConvNormAct(w, w * 2, 2),
|
| 276 |
+
ResBlock(w * 2),
|
| 277 |
+
ConvNormAct(w * 2, w * 3, 2),
|
| 278 |
+
ResBlock(w * 3),
|
| 279 |
+
ConvNormAct(w * 3, w * 4, 2),
|
| 280 |
+
nn.AdaptiveAvgPool2d(1),
|
| 281 |
+
nn.Flatten(),
|
| 282 |
+
)
|
| 283 |
+
self.context_net = nn.Sequential(
|
| 284 |
+
nn.Linear(cfg.context_dim + cfg.semantic_dim + 1, cfg.hidden_dim // 2),
|
| 285 |
+
nn.LayerNorm(cfg.hidden_dim // 2),
|
| 286 |
+
nn.GELU(),
|
| 287 |
+
nn.Linear(cfg.hidden_dim // 2, cfg.hidden_dim // 2),
|
| 288 |
+
nn.GELU(),
|
| 289 |
+
)
|
| 290 |
+
image_dim = w * 4
|
| 291 |
+
self.fuse = nn.Sequential(
|
| 292 |
+
nn.Linear(image_dim + cfg.hidden_dim // 2, cfg.hidden_dim),
|
| 293 |
+
nn.LayerNorm(cfg.hidden_dim),
|
| 294 |
+
nn.GELU(),
|
| 295 |
+
nn.Linear(cfg.hidden_dim, cfg.hidden_dim),
|
| 296 |
+
nn.GELU(),
|
| 297 |
+
)
|
| 298 |
+
self.sem_mu = nn.Linear(cfg.hidden_dim, cfg.semantic_dim)
|
| 299 |
+
self.sem_logvar = nn.Linear(cfg.hidden_dim, cfg.semantic_dim)
|
| 300 |
+
self.shape_mu = nn.Linear(cfg.hidden_dim, cfg.shape_dim)
|
| 301 |
+
self.shape_logvar = nn.Linear(cfg.hidden_dim, cfg.shape_dim)
|
| 302 |
+
|
| 303 |
+
# Start from the analytic center and nearly-zero free shape code.
|
| 304 |
+
nn.init.zeros_(self.sem_mu.weight)
|
| 305 |
+
nn.init.zeros_(self.sem_mu.bias)
|
| 306 |
+
nn.init.constant_(self.sem_logvar.bias, -4.0)
|
| 307 |
+
nn.init.normal_(self.shape_mu.weight, std=0.01)
|
| 308 |
+
nn.init.zeros_(self.shape_mu.bias)
|
| 309 |
+
nn.init.constant_(self.shape_logvar.bias, -3.0)
|
| 310 |
+
|
| 311 |
+
def forward(
|
| 312 |
+
self,
|
| 313 |
+
canonical: torch.Tensor,
|
| 314 |
+
analytic_sem: torch.Tensor,
|
| 315 |
+
context: torch.Tensor,
|
| 316 |
+
exist_prob: torch.Tensor,
|
| 317 |
+
) -> Dict[str, torch.Tensor]:
|
| 318 |
+
B, K = canonical.shape[:2]
|
| 319 |
+
img = self.image_net(canonical.reshape(B * K, *canonical.shape[2:]))
|
| 320 |
+
ctx_in = torch.cat([
|
| 321 |
+
context.reshape(B * K, -1),
|
| 322 |
+
analytic_sem.reshape(B * K, -1),
|
| 323 |
+
exist_prob.reshape(B * K, 1),
|
| 324 |
+
], dim=-1)
|
| 325 |
+
ctx = self.context_net(ctx_in)
|
| 326 |
+
h = self.fuse(torch.cat([img, ctx], dim=-1))
|
| 327 |
+
sem_mu = self.sem_mu(h).reshape(B, K, -1)
|
| 328 |
+
sem_logvar = self.sem_logvar(h).clamp(-8.0, 3.0).reshape(B, K, -1)
|
| 329 |
+
shape_mu = self.shape_mu(h).reshape(B, K, -1)
|
| 330 |
+
shape_logvar = self.shape_logvar(h).clamp(-8.0, 3.0).reshape(B, K, -1)
|
| 331 |
return {
|
| 332 |
+
"sem_delta_mu": sem_mu,
|
| 333 |
+
"sem_delta_logvar": sem_logvar,
|
| 334 |
+
"shape_mu": shape_mu,
|
| 335 |
+
"shape_logvar": shape_logvar,
|
|
|
|
|
|
|
| 336 |
}
|
| 337 |
|
| 338 |
+
|
| 339 |
+
class SemanticRenderer(nn.Module):
|
| 340 |
+
"""Parameter-free generalized elliptical renderer from corrected semantics."""
|
| 341 |
+
def __init__(self, cfg: V5AwareACSSVAEConfig):
|
| 342 |
+
super().__init__()
|
| 343 |
+
self.cfg = cfg
|
| 344 |
+
f = torch.linspace(0, 1, cfg.n_freqs).view(1, 1, cfg.n_freqs, 1)
|
| 345 |
+
t = torch.linspace(0, 2 * math.pi, cfg.n_dirs + 1)[:cfg.n_dirs].view(1, 1, 1, cfg.n_dirs)
|
| 346 |
+
self.register_buffer("fgrid", f)
|
| 347 |
+
self.register_buffer("tgrid", t)
|
| 348 |
+
|
| 349 |
+
def forward(self, sem: torch.Tensor) -> torch.Tensor:
|
| 350 |
+
cfg = self.cfg
|
| 351 |
+
eps = cfg.eps
|
| 352 |
+
loge, fp = sem[..., 0], sem[..., 1]
|
| 353 |
+
theta = torch.atan2(sem[..., 2], sem[..., 3])
|
| 354 |
+
sf = sem[..., 4].clamp(cfg.min_spread_f, cfg.max_spread_f)
|
| 355 |
+
st = sem[..., 5].clamp(cfg.min_spread_t, cfg.max_spread_t)
|
| 356 |
+
rho = sem[..., 6].clamp(-cfg.max_abs_rho, cfg.max_abs_rho)
|
| 357 |
+
skew_f = sem[..., 7].clamp(-cfg.max_abs_skew, cfg.max_abs_skew)
|
| 358 |
+
skew_t = sem[..., 8].clamp(-cfg.max_abs_skew, cfg.max_abs_skew)
|
| 359 |
+
|
| 360 |
+
df = (self.fgrid - fp[:, :, None, None]) / sf[:, :, None, None].clamp_min(1e-3)
|
| 361 |
+
dt = _wrap_angle(self.tgrid - theta[:, :, None, None])
|
| 362 |
+
dt = dt / (math.pi * st[:, :, None, None].clamp_min(1e-3))
|
| 363 |
+
den = (1.0 - rho.square()).clamp_min(0.08)
|
| 364 |
+
q = (df.square() + dt.square() - 2.0 * rho[:, :, None, None] * df * dt) / den[:, :, None, None]
|
| 365 |
+
shape = torch.exp(-0.5 * q.clamp_max(60.0))
|
| 366 |
+
asym = torch.exp(
|
| 367 |
+
0.28 * skew_f[:, :, None, None] * torch.tanh(df)
|
| 368 |
+
+ 0.28 * skew_t[:, :, None, None] * torch.tanh(dt)
|
| 369 |
+
).clamp(0.15, 6.0)
|
| 370 |
+
shape = shape * asym
|
| 371 |
+
shape = shape / shape.sum(dim=(-2, -1), keepdim=True).clamp_min(eps)
|
| 372 |
+
mass = torch.expm1(loge * cfg.energy_log_den).clamp_min(0.0)
|
| 373 |
+
return shape * mass[:, :, None, None]
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
class ShapeResidualNet(nn.Module):
|
| 377 |
+
def __init__(self, cfg: V5AwareACSSVAEConfig):
|
| 378 |
+
super().__init__()
|
| 379 |
+
self.cfg = cfg
|
| 380 |
+
w = cfg.decoder_width
|
| 381 |
+
self.fc = nn.Sequential(
|
| 382 |
+
nn.Linear(cfg.semantic_dim + cfg.shape_dim, cfg.hidden_dim),
|
| 383 |
+
nn.GELU(),
|
| 384 |
+
nn.Linear(cfg.hidden_dim, w * 6 * 9),
|
| 385 |
+
nn.GELU(),
|
| 386 |
+
)
|
| 387 |
+
self.net = nn.Sequential(
|
| 388 |
+
ConvNormAct(w, w, 1),
|
| 389 |
+
ResBlock(w),
|
| 390 |
+
nn.Upsample(size=(12, 18), mode="bilinear", align_corners=False),
|
| 391 |
+
ConvNormAct(w, w, 1),
|
| 392 |
+
ResBlock(w),
|
| 393 |
+
nn.Upsample(size=(24, 36), mode="bilinear", align_corners=False),
|
| 394 |
+
ConvNormAct(w, w // 2, 1),
|
| 395 |
+
ResBlock(w // 2),
|
| 396 |
+
nn.Upsample(size=(48, 72), mode="bilinear", align_corners=False),
|
| 397 |
+
ConvNormAct(w // 2, w // 2, 1),
|
| 398 |
+
nn.Conv2d(w // 2, 1, 3, padding=1),
|
| 399 |
+
)
|
| 400 |
+
|
| 401 |
+
def field(self, sem: torch.Tensor, zshape: torch.Tensor) -> torch.Tensor:
|
| 402 |
+
B, K = sem.shape[:2]
|
| 403 |
+
x = torch.cat([sem, zshape], dim=-1).reshape(B * K, -1)
|
| 404 |
+
h = self.fc(x).view(B * K, self.cfg.decoder_width, 6, 9)
|
| 405 |
+
out = self.net(h)[:, :, : self.cfg.n_freqs, : self.cfg.n_dirs]
|
| 406 |
+
return out.reshape(B, K, self.cfg.n_freqs, self.cfg.n_dirs)
|
| 407 |
+
|
| 408 |
+
def forward(self, sem: torch.Tensor, zshape: torch.Tensor) -> torch.Tensor:
|
| 409 |
+
raw = self.field(sem, zshape)
|
| 410 |
+
zero = self.field(sem, torch.zeros_like(zshape))
|
| 411 |
+
return self.cfg.shape_log_residual_scale * torch.tanh(raw - zero)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
class V5AwareStructuredSemanticVAE(nn.Module):
|
| 415 |
+
def __init__(self, cfg: Optional[V5AwareACSSVAEConfig] = None):
|
| 416 |
+
super().__init__()
|
| 417 |
+
self.cfg = cfg or V5AwareACSSVAEConfig()
|
| 418 |
+
self.canonicalizer = Canonicalizer(self.cfg)
|
| 419 |
+
self.encoder = SlotEncoder(self.cfg)
|
| 420 |
+
self.semantic_renderer = SemanticRenderer(self.cfg)
|
| 421 |
+
self.shape_decoder = ShapeResidualNet(self.cfg)
|
| 422 |
+
|
| 423 |
+
def apply_semantic_delta(self, base: torch.Tensor, delta: torch.Tensor) -> torch.Tensor:
|
| 424 |
+
c = self.cfg
|
| 425 |
+
out = base.clone()
|
| 426 |
+
out[..., 0] = (base[..., 0] + c.delta_loge * torch.tanh(delta[..., 0])).clamp_min(0.0)
|
| 427 |
+
out[..., 1] = (base[..., 1] + c.delta_f * torch.tanh(delta[..., 1])).clamp(0.0, 1.0)
|
| 428 |
+
|
| 429 |
+
theta0 = torch.atan2(base[..., 2], base[..., 3])
|
| 430 |
+
# Use both sin/cos correction channels to form one stable tangent-angle correction.
|
| 431 |
+
dtheta = c.delta_angle_rad * torch.tanh(0.7071 * (delta[..., 2] - delta[..., 3]))
|
| 432 |
+
theta = theta0 + dtheta
|
| 433 |
+
out[..., 2] = torch.sin(theta)
|
| 434 |
+
out[..., 3] = torch.cos(theta)
|
| 435 |
+
|
| 436 |
+
out[..., 4] = (
|
| 437 |
+
base[..., 4].clamp_min(c.min_spread_f)
|
| 438 |
+
* torch.exp(c.delta_log_spread * torch.tanh(delta[..., 4]))
|
| 439 |
+
).clamp(c.min_spread_f, c.max_spread_f)
|
| 440 |
+
out[..., 5] = (
|
| 441 |
+
base[..., 5].clamp_min(c.min_spread_t)
|
| 442 |
+
* torch.exp(c.delta_log_spread * torch.tanh(delta[..., 5]))
|
| 443 |
+
).clamp(c.min_spread_t, c.max_spread_t)
|
| 444 |
+
out[..., 6] = torch.tanh(
|
| 445 |
+
_atanh_safe(base[..., 6], c.eps)
|
| 446 |
+
+ c.delta_rho_logit * torch.tanh(delta[..., 6])
|
| 447 |
+
).clamp(-c.max_abs_rho, c.max_abs_rho)
|
| 448 |
+
out[..., 7] = (
|
| 449 |
+
base[..., 7] + c.delta_skew * torch.tanh(delta[..., 7])
|
| 450 |
+
).clamp(-c.max_abs_skew, c.max_abs_skew)
|
| 451 |
+
out[..., 8] = (
|
| 452 |
+
base[..., 8] + c.delta_skew * torch.tanh(delta[..., 8])
|
| 453 |
+
).clamp(-c.max_abs_skew, c.max_abs_skew)
|
| 454 |
+
return out
|
| 455 |
+
|
| 456 |
+
def decode_from_codes(
|
| 457 |
+
self,
|
| 458 |
+
semantics: torch.Tensor,
|
| 459 |
+
shape_code: torch.Tensor,
|
| 460 |
+
exist_prob: Optional[torch.Tensor] = None,
|
| 461 |
+
) -> Dict[str, torch.Tensor]:
|
| 462 |
+
base = self.semantic_renderer(semantics)
|
| 463 |
+
residual_log = self.shape_decoder(semantics, shape_code)
|
| 464 |
+
# Multiplicative residual in log domain, followed by exact energy renormalization.
|
| 465 |
+
part_hat = (base + self.cfg.eps) * torch.exp(residual_log)
|
| 466 |
+
target_mass = torch.expm1(semantics[..., 0] * self.cfg.energy_log_den).clamp_min(0.0)
|
| 467 |
+
part_hat = part_hat * (
|
| 468 |
+
target_mass[:, :, None, None]
|
| 469 |
+
/ part_hat.sum(dim=(-2, -1), keepdim=True).clamp_min(self.cfg.eps)
|
| 470 |
+
)
|
| 471 |
+
if exist_prob is not None:
|
| 472 |
+
part_hat = part_hat * exist_prob[:, :, None, None]
|
| 473 |
+
base = base * exist_prob[:, :, None, None]
|
| 474 |
+
return {
|
| 475 |
+
"semantic_base": base,
|
| 476 |
+
"shape_residual_log": residual_log,
|
| 477 |
+
"part_hat": part_hat,
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
def forward(
|
| 481 |
+
self,
|
| 482 |
+
energy: torch.Tensor,
|
| 483 |
+
mask_prob: torch.Tensor,
|
| 484 |
+
core_prob: torch.Tensor,
|
| 485 |
+
support_prob: torch.Tensor,
|
| 486 |
+
exist_prob: torch.Tensor,
|
| 487 |
+
slot_context: torch.Tensor,
|
| 488 |
+
stochastic: bool = True,
|
| 489 |
+
shape_code_override: Optional[torch.Tensor] = None,
|
| 490 |
+
semantic_delta_override: Optional[torch.Tensor] = None,
|
| 491 |
+
) -> Dict[str, torch.Tensor]:
|
| 492 |
+
analytic = analytic_semantics(
|
| 493 |
+
energy,
|
| 494 |
+
mask_prob,
|
| 495 |
+
core_prob=core_prob,
|
| 496 |
+
eps=self.cfg.eps,
|
| 497 |
+
energy_log_den=self.cfg.energy_log_den,
|
| 498 |
+
)
|
| 499 |
+
canonical = self.canonicalizer(
|
| 500 |
+
energy,
|
| 501 |
+
mask_prob,
|
| 502 |
+
core_prob,
|
| 503 |
+
support_prob,
|
| 504 |
+
analytic,
|
| 505 |
+
)
|
| 506 |
+
enc = self.encoder(canonical, analytic, slot_context, exist_prob)
|
| 507 |
+
sem_delta = (
|
| 508 |
+
semantic_delta_override
|
| 509 |
+
if semantic_delta_override is not None
|
| 510 |
+
else reparameterize(enc["sem_delta_mu"], enc["sem_delta_logvar"], stochastic)
|
| 511 |
+
)
|
| 512 |
+
zshape = (
|
| 513 |
+
shape_code_override
|
| 514 |
+
if shape_code_override is not None
|
| 515 |
+
else reparameterize(enc["shape_mu"], enc["shape_logvar"], stochastic)
|
| 516 |
+
)
|
| 517 |
+
corrected = self.apply_semantic_delta(analytic, sem_delta)
|
| 518 |
+
dec = self.decode_from_codes(corrected, zshape, exist_prob=exist_prob)
|
| 519 |
+
return {
|
| 520 |
+
"analytic_semantics": analytic,
|
| 521 |
+
"corrected_semantics": corrected,
|
| 522 |
+
"semantic_delta": sem_delta,
|
| 523 |
+
"shape_code": zshape,
|
| 524 |
+
"canonical": canonical,
|
| 525 |
+
**enc,
|
| 526 |
+
**dec,
|
| 527 |
+
}
|
| 528 |
+
|
| 529 |
+
@property
|
| 530 |
+
def token_dim_per_wave(self) -> int:
|
| 531 |
+
return self.cfg.semantic_dim + self.cfg.shape_dim
|
| 532 |
+
|
| 533 |
+
def num_params(self) -> int:
|
| 534 |
+
return sum(p.numel() for p in self.parameters())
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
def kl_standard_normal(mu: torch.Tensor, logvar: torch.Tensor, weight: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 538 |
+
kl = -0.5 * (1.0 + logvar - mu.square() - logvar.exp()).sum(dim=-1)
|
| 539 |
+
if weight is None:
|
| 540 |
+
return kl.mean()
|
| 541 |
+
w = weight.float()
|
| 542 |
+
return (kl * w).sum() / w.sum().clamp_min(1.0)
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
def covariance_penalty(x: torch.Tensor, weight: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 546 |
+
"""Off-diagonal covariance penalty for [B,K,D] latent tensors."""
|
| 547 |
+
D = x.shape[-1]
|
| 548 |
+
flat = x.reshape(-1, D)
|
| 549 |
+
if weight is not None:
|
| 550 |
+
w = weight.reshape(-1).float()
|
| 551 |
+
keep = w > 0.25
|
| 552 |
+
flat = flat[keep]
|
| 553 |
+
if flat.shape[0] < max(4, D):
|
| 554 |
+
return x.new_zeros(())
|
| 555 |
+
flat = flat - flat.mean(dim=0, keepdim=True)
|
| 556 |
+
cov = flat.T @ flat / max(flat.shape[0] - 1, 1)
|
| 557 |
+
off = cov - torch.diag(torch.diag(cov))
|
| 558 |
+
return off.square().mean()
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def semantic_distance(pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
|
| 562 |
+
"""Per-slot interpretable semantic distance with circular direction handling."""
|
| 563 |
+
d = []
|
| 564 |
+
d.append((pred[..., 0] - target[..., 0]).abs())
|
| 565 |
+
d.append(2.0 * (pred[..., 1] - target[..., 1]).abs())
|
| 566 |
+
tp = torch.atan2(pred[..., 2], pred[..., 3])
|
| 567 |
+
tt = torch.atan2(target[..., 2], target[..., 3])
|
| 568 |
+
d.append(_wrap_angle(tp - tt).abs() / math.pi)
|
| 569 |
+
d.append(2.0 * (pred[..., 4] - target[..., 4]).abs())
|
| 570 |
+
d.append(1.5 * (pred[..., 5] - target[..., 5]).abs())
|
| 571 |
+
d.append(0.5 * (pred[..., 6] - target[..., 6]).abs())
|
| 572 |
+
d.append(0.15 * (pred[..., 7] - target[..., 7]).abs())
|
| 573 |
+
d.append(0.15 * (pred[..., 8] - target[..., 8]).abs())
|
| 574 |
+
return torch.stack(d, dim=-1).mean(dim=-1)
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def marginal_l1(pred: torch.Tensor, target: torch.Tensor, eps: float = 1e-6) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 578 |
+
p = pred.clamp_min(0)
|
| 579 |
+
t = target.clamp_min(0)
|
| 580 |
+
pf = p.sum(dim=-1); tf = t.sum(dim=-1)
|
| 581 |
+
pt = p.sum(dim=-2); tt = t.sum(dim=-2)
|
| 582 |
+
pf = pf / pf.sum(dim=-1, keepdim=True).clamp_min(eps)
|
| 583 |
+
tf = tf / tf.sum(dim=-1, keepdim=True).clamp_min(eps)
|
| 584 |
+
pt = pt / pt.sum(dim=-1, keepdim=True).clamp_min(eps)
|
| 585 |
+
tt = tt / tt.sum(dim=-1, keepdim=True).clamp_min(eps)
|
| 586 |
+
return (pf - tf).abs().mean(), (pt - tt).abs().mean()
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def reconstruction_terms(pred: torch.Tensor, target: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6) -> Dict[str, torch.Tensor]:
|
| 590 |
+
w = weight[:, :, None, None].float()
|
| 591 |
+
denom = w.sum().clamp_min(1.0)
|
| 592 |
+
mse_slot = (pred - target).square().mean(dim=(-2, -1))
|
| 593 |
+
log_slot = (
|
| 594 |
+
torch.log1p(20.0 * pred.clamp_min(0))
|
| 595 |
+
- torch.log1p(20.0 * target.clamp_min(0))
|
| 596 |
+
).abs().mean(dim=(-2, -1))
|
| 597 |
+
mass_p = pred.sum(dim=(-2, -1))
|
| 598 |
+
mass_t = target.sum(dim=(-2, -1))
|
| 599 |
+
energy_slot = (mass_p - mass_t).abs() / mass_t.clamp_min(eps)
|
| 600 |
+
mse = (mse_slot * weight).sum() / denom
|
| 601 |
+
log_l1 = (log_slot * weight).sum() / denom
|
| 602 |
+
energy = (energy_slot * weight).sum() / denom
|
| 603 |
+
freq, direction = marginal_l1(pred * w, target * w, eps=eps)
|
| 604 |
+
return {
|
| 605 |
+
"mse": mse,
|
| 606 |
+
"log_l1": log_l1,
|
| 607 |
+
"energy": energy,
|
| 608 |
+
"freq": freq,
|
| 609 |
+
"direction": direction,
|
| 610 |
+
}
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
__all__ = [
|
| 614 |
+
"V5AwareACSSVAEConfig",
|
| 615 |
+
"V5AwareStructuredSemanticVAE",
|
| 616 |
+
"analytic_semantics",
|
| 617 |
+
"kl_standard_normal",
|
| 618 |
+
"covariance_penalty",
|
| 619 |
+
"semantic_distance",
|
| 620 |
+
"reconstruction_terms",
|
| 621 |
+
"marginal_l1",
|
| 622 |
+
]
|
| 623 |
+
|
| 624 |
+
# =============================================================================
|
| 625 |
+
# ICWDS Stage-2 hybrid codec components (C3-D / C3-F / C4-B)
|
| 626 |
+
# =============================================================================
|
| 627 |
+
|
| 628 |
+
NF = 47
|
| 629 |
+
ND = 72
|
| 630 |
+
K_MAX = 6
|
| 631 |
+
|
| 632 |
+
class PrototypeSelector(nn.Module):
|
| 633 |
+
def __init__(self, input_dim, n_codes, width=192, dropout=0.10):
|
| 634 |
+
super().__init__()
|
| 635 |
+
self.norm = nn.LayerNorm(input_dim)
|
| 636 |
+
self.fc1 = nn.Linear(input_dim, width)
|
| 637 |
+
self.fc2 = nn.Linear(width, width)
|
| 638 |
+
self.fc3 = nn.Linear(width, width)
|
| 639 |
+
self.drop = nn.Dropout(dropout)
|
| 640 |
+
self.head = nn.Linear(width, n_codes)
|
| 641 |
+
|
| 642 |
+
def forward(self, x):
|
| 643 |
+
x = self.norm(x)
|
| 644 |
+
h = F.gelu(self.fc1(x))
|
| 645 |
+
r = h
|
| 646 |
+
h = self.drop(F.gelu(self.fc2(h)))
|
| 647 |
+
h = F.gelu(self.fc3(h) + r)
|
| 648 |
+
return self.head(h)
|
| 649 |
+
|
| 650 |
+
class PeriodicConv2d(nn.Module):
|
| 651 |
+
def __init__(self, ci, co, kernel=3, stride=1, bias=False):
|
| 652 |
+
super().__init__()
|
| 653 |
+
self.pad = kernel // 2
|
| 654 |
+
self.conv = nn.Conv2d(ci, co, kernel_size=kernel, stride=stride, padding=0, bias=bias)
|
| 655 |
+
|
| 656 |
+
def forward(self, x):
|
| 657 |
+
p = self.pad
|
| 658 |
+
x = F.pad(x, (p, p, 0, 0), mode="circular")
|
| 659 |
+
x = F.pad(x, (0, 0, p, p), mode="replicate")
|
| 660 |
+
return self.conv(x)
|
| 661 |
+
|
| 662 |
+
class PResBlock(nn.Module):
|
| 663 |
+
def __init__(self, ch):
|
| 664 |
+
super().__init__()
|
| 665 |
+
self.c1 = PeriodicConv2d(ch, ch, 3, 1, False)
|
| 666 |
+
self.n1 = nn.GroupNorm(min(8, ch), ch)
|
| 667 |
+
self.c2 = PeriodicConv2d(ch, ch, 3, 1, False)
|
| 668 |
+
self.n2 = nn.GroupNorm(min(8, ch), ch)
|
| 669 |
+
|
| 670 |
+
def forward(self, x):
|
| 671 |
+
h = F.gelu(self.n1(self.c1(x)))
|
| 672 |
+
h = self.n2(self.c2(h))
|
| 673 |
+
return F.gelu(x + h)
|
| 674 |
+
|
| 675 |
+
class NestedBitEncoder(nn.Module):
|
| 676 |
+
def __init__(self, max_bits, width=32):
|
| 677 |
+
super().__init__()
|
| 678 |
+
w = width
|
| 679 |
+
self.stem = nn.Sequential(
|
| 680 |
+
PeriodicConv2d(4, w, 3, 2, False),
|
| 681 |
+
nn.GroupNorm(min(8, w), w), nn.GELU(), PResBlock(w),
|
| 682 |
+
PeriodicConv2d(w, w*2, 3, 2, False),
|
| 683 |
+
nn.GroupNorm(min(8, w*2), w*2), nn.GELU(), PResBlock(w*2),
|
| 684 |
+
PeriodicConv2d(w*2, w*3, 3, 2, False),
|
| 685 |
+
nn.GroupNorm(min(8, w*3), w*3), nn.GELU(), PResBlock(w*3),
|
| 686 |
+
)
|
| 687 |
+
ch = w * 3
|
| 688 |
+
self.attn = nn.Conv2d(ch, 1, 1)
|
| 689 |
+
self.head = nn.Sequential(
|
| 690 |
+
nn.Linear(ch * 3, ch * 2),
|
| 691 |
+
nn.LayerNorm(ch * 2),
|
| 692 |
+
nn.GELU(),
|
| 693 |
+
nn.Linear(ch * 2, max_bits),
|
| 694 |
+
)
|
| 695 |
+
|
| 696 |
+
def forward(self, residual_target, prior, gate):
|
| 697 |
+
x = torch.stack([residual_target, residual_target.abs(), prior, gate], dim=1)
|
| 698 |
+
h = self.stem(x)
|
| 699 |
+
flat = h.flatten(2)
|
| 700 |
+
attn = torch.softmax(self.attn(h).flatten(2), dim=-1)
|
| 701 |
+
ap = (flat * attn).sum(dim=-1)
|
| 702 |
+
mx = F.adaptive_max_pool2d(h, 1).flatten(1)
|
| 703 |
+
av = F.adaptive_avg_pool2d(h, 1).flatten(1)
|
| 704 |
+
return self.head(torch.cat([ap, mx, av], dim=-1))
|
| 705 |
+
|
| 706 |
+
def ste_binary(logits):
|
| 707 |
+
soft = torch.tanh(logits)
|
| 708 |
+
hard = torch.where(logits >= 0, torch.ones_like(logits), -torch.ones_like(logits))
|
| 709 |
+
q = soft + (hard - soft).detach()
|
| 710 |
+
return q, soft
|
| 711 |
+
|
| 712 |
+
def prefix_mask(n_bits, max_bits, device, dtype):
|
| 713 |
+
if int(n_bits) < 0 or int(n_bits) > int(max_bits):
|
| 714 |
+
raise ValueError((n_bits, max_bits))
|
| 715 |
+
return (torch.arange(max_bits, device=device) < int(n_bits)).to(dtype=dtype)[None]
|
| 716 |
+
|
| 717 |
+
class CondPyramid(nn.Module):
|
| 718 |
+
def __init__(self, width=32):
|
| 719 |
+
super().__init__()
|
| 720 |
+
w = width
|
| 721 |
+
self.c0 = nn.Sequential(
|
| 722 |
+
PeriodicConv2d(2, w, 3, 1, False),
|
| 723 |
+
nn.GroupNorm(min(8, w), w), nn.GELU(),
|
| 724 |
+
)
|
| 725 |
+
self.c1 = nn.Sequential(
|
| 726 |
+
PeriodicConv2d(w, w*2, 3, 2, False),
|
| 727 |
+
nn.GroupNorm(min(8, w*2), w*2), nn.GELU(),
|
| 728 |
+
)
|
| 729 |
+
self.c2 = nn.Sequential(
|
| 730 |
+
PeriodicConv2d(w*2, w*3, 3, 2, False),
|
| 731 |
+
nn.GroupNorm(min(8, w*3), w*3), nn.GELU(),
|
| 732 |
+
)
|
| 733 |
+
self.c3 = nn.Sequential(
|
| 734 |
+
PeriodicConv2d(w*3, w*3, 3, 2, False),
|
| 735 |
+
nn.GroupNorm(min(8, w*3), w*3), nn.GELU(),
|
| 736 |
+
)
|
| 737 |
+
|
| 738 |
+
def forward(self, prior, gate):
|
| 739 |
+
x = torch.stack([prior, gate], dim=1)
|
| 740 |
+
c0 = self.c0(x)
|
| 741 |
+
c1 = self.c1(c0)
|
| 742 |
+
c2 = self.c2(c1)
|
| 743 |
+
c3 = self.c3(c2)
|
| 744 |
+
return c0, c1, c2, c3
|
| 745 |
+
|
| 746 |
+
class CodeFiLM(nn.Module):
|
| 747 |
+
"""Near-identity code injection. Final layer is zero-initialized."""
|
| 748 |
+
def __init__(self, max_bits, channels):
|
| 749 |
+
super().__init__()
|
| 750 |
+
hid = max(48, channels)
|
| 751 |
+
self.net = nn.Sequential(
|
| 752 |
+
nn.Linear(max_bits + 1, hid),
|
| 753 |
+
nn.GELU(),
|
| 754 |
+
nn.Linear(hid, channels * 2),
|
| 755 |
+
)
|
| 756 |
+
nn.init.zeros_(self.net[-1].weight)
|
| 757 |
+
nn.init.zeros_(self.net[-1].bias)
|
| 758 |
+
|
| 759 |
+
def forward(self, h, qbits, bit_fraction):
|
| 760 |
+
B = qbits.shape[0]
|
| 761 |
+
bf = qbits.new_full((B, 1), float(bit_fraction))
|
| 762 |
+
gamma, beta = self.net(torch.cat([qbits, bf], dim=-1)).chunk(2, dim=-1)
|
| 763 |
+
gamma = 0.20 * torch.tanh(gamma)[:, :, None, None]
|
| 764 |
+
beta = 0.20 * torch.tanh(beta)[:, :, None, None]
|
| 765 |
+
return h * (1.0 + gamma) + beta
|
| 766 |
+
|
| 767 |
+
class DetailAwareResidualDecoder(nn.Module):
|
| 768 |
+
def __init__(self, max_bits, width=32, branch="intra"):
|
| 769 |
+
super().__init__()
|
| 770 |
+
self.max_bits = int(max_bits)
|
| 771 |
+
self.branch = str(branch)
|
| 772 |
+
w = width
|
| 773 |
+
self.cond = CondPyramid(w)
|
| 774 |
+
self.bit_fc = nn.Sequential(nn.Linear(max_bits, w*3*6*9), nn.GELU())
|
| 775 |
+
self.b3 = PResBlock(w*3)
|
| 776 |
+
self.p2 = nn.Conv2d(w*3, w*3, 1)
|
| 777 |
+
self.b2 = PResBlock(w*3)
|
| 778 |
+
self.p1 = nn.Conv2d(w*3, w*2, 1)
|
| 779 |
+
self.b1 = PResBlock(w*2)
|
| 780 |
+
self.p0 = nn.Conv2d(w*2, w, 1)
|
| 781 |
+
self.b0 = PResBlock(w)
|
| 782 |
+
self.out = PeriodicConv2d(w, 1, 3, 1, True)
|
| 783 |
+
self.scale = nn.Parameter(torch.tensor(0.35))
|
| 784 |
+
|
| 785 |
+
# New C3-F modules.
|
| 786 |
+
self.film3 = CodeFiLM(max_bits, w*3)
|
| 787 |
+
self.film2 = CodeFiLM(max_bits, w*3)
|
| 788 |
+
self.film1 = CodeFiLM(max_bits, w*2)
|
| 789 |
+
self.film0 = CodeFiLM(max_bits, w)
|
| 790 |
+
self.detail_block = PResBlock(w)
|
| 791 |
+
self.detail_out = PeriodicConv2d(w, 1, 3, 1, True)
|
| 792 |
+
nn.init.zeros_(self.detail_out.conv.weight)
|
| 793 |
+
if self.detail_out.conv.bias is not None:
|
| 794 |
+
nn.init.zeros_(self.detail_out.conv.bias)
|
| 795 |
+
self.detail_scale = nn.Parameter(torch.tensor(-2.0))
|
| 796 |
+
|
| 797 |
+
def forward(self, qbits, prior, gate, n_bits=None):
|
| 798 |
+
B = prior.shape[0]
|
| 799 |
+
if n_bits is None:
|
| 800 |
+
n_bits = int((qbits.abs().sum(dim=0) > 0).sum().item())
|
| 801 |
+
frac = float(n_bits) / float(self.max_bits)
|
| 802 |
+
c0, c1, c2, c3 = self.cond(prior, gate)
|
| 803 |
+
|
| 804 |
+
h = self.bit_fc(qbits).view(B, c3.shape[1], 6, 9)
|
| 805 |
+
h = self.b3(h + c3)
|
| 806 |
+
h = self.film3(h, qbits, frac)
|
| 807 |
+
|
| 808 |
+
h = F.interpolate(h, size=c2.shape[-2:], mode="bilinear", align_corners=False)
|
| 809 |
+
h = self.b2(self.p2(h) + c2)
|
| 810 |
+
h = self.film2(h, qbits, frac)
|
| 811 |
+
|
| 812 |
+
h = F.interpolate(h, size=c1.shape[-2:], mode="bilinear", align_corners=False)
|
| 813 |
+
h = self.b1(self.p1(h) + c1)
|
| 814 |
+
h = self.film1(h, qbits, frac)
|
| 815 |
+
|
| 816 |
+
h = F.interpolate(h, size=(48, 72), mode="bilinear", align_corners=False)
|
| 817 |
+
c0p = F.pad(c0, (0, 0, 0, 1), mode="replicate")
|
| 818 |
+
h = self.b0(self.p0(h) + c0p)
|
| 819 |
+
h = self.film0(h, qbits, frac)
|
| 820 |
+
|
| 821 |
+
base = self.out(h)[:, 0, :NF, :ND]
|
| 822 |
+
detail = self.detail_out(self.detail_block(h))[:, 0, :NF, :ND]
|
| 823 |
+
# High-pass detail residual; direction is periodic, frequency is replicated.
|
| 824 |
+
dp = F.pad(detail[:, None], (1, 1, 0, 0), mode="circular")
|
| 825 |
+
dp = F.pad(dp, (0, 0, 1, 1), mode="replicate")
|
| 826 |
+
blur = F.avg_pool2d(dp, 3, stride=1)[:, 0]
|
| 827 |
+
detail_hp = detail - blur
|
| 828 |
+
raw = torch.tanh(base + torch.sigmoid(self.detail_scale) * detail_hp)
|
| 829 |
+
|
| 830 |
+
amp = 0.65 * torch.sigmoid(self.scale)
|
| 831 |
+
spatial = (0.08 + 0.92 * gate) if self.branch == "intra" else (1.0 - 0.72 * gate)
|
| 832 |
+
return amp * raw * spatial
|
| 833 |
+
|
| 834 |
+
class AdaptiveDualResidualCodec(nn.Module):
|
| 835 |
+
def __init__(self, max_bits=24, enc_width=32, dec_width=32):
|
| 836 |
+
super().__init__()
|
| 837 |
+
self.max_bits = int(max_bits)
|
| 838 |
+
self.intra_encoder = NestedBitEncoder(max_bits, enc_width)
|
| 839 |
+
self.bg_encoder = NestedBitEncoder(max_bits, enc_width)
|
| 840 |
+
self.intra_decoder = DetailAwareResidualDecoder(max_bits, dec_width, branch="intra")
|
| 841 |
+
self.bg_decoder = DetailAwareResidualDecoder(max_bits, dec_width, branch="bg")
|
| 842 |
+
|
| 843 |
+
def encode(self, prior, gate, intra_target, bg_target):
|
| 844 |
+
li = self.intra_encoder(intra_target, prior, gate)
|
| 845 |
+
lb = self.bg_encoder(bg_target, prior, gate)
|
| 846 |
+
return li, lb
|
| 847 |
+
|
| 848 |
+
def decode_logits(self, logits, n_bits, prior, gate, branch, q_override=None):
|
| 849 |
+
if int(n_bits) == 0:
|
| 850 |
+
return torch.zeros_like(prior), None, None
|
| 851 |
+
q, soft = ste_binary(logits)
|
| 852 |
+
mask = prefix_mask(n_bits, self.max_bits, logits.device, logits.dtype)
|
| 853 |
+
q = q * mask
|
| 854 |
+
if q_override is not None:
|
| 855 |
+
q = q_override * mask
|
| 856 |
+
dec = self.intra_decoder if branch == "intra" else self.bg_decoder
|
| 857 |
+
pred = dec(q, prior, gate, n_bits=int(n_bits))
|
| 858 |
+
return pred, q, soft
|
| 859 |
+
|
| 860 |
+
def decode_pair(self, li, lb, prior, gate, intra_bits, bg_bits):
|
| 861 |
+
ri, qi, si = self.decode_logits(li, intra_bits, prior, gate, "intra")
|
| 862 |
+
rb, qb, sb = self.decode_logits(lb, bg_bits, prior, gate, "bg")
|
| 863 |
+
return {"ri": ri, "rb": rb, "qi": qi, "qb": qb, "si": si, "sb": sb}
|
| 864 |
+
|
| 865 |
+
def forward(self, prior, gate, intra_target, bg_target, intra_bits, bg_bits):
|
| 866 |
+
li, lb = self.encode(prior, gate, intra_target, bg_target)
|
| 867 |
+
out = self.decode_pair(li, lb, prior, gate, intra_bits, bg_bits)
|
| 868 |
+
out.update({"li": li, "lb": lb})
|
| 869 |
+
return out
|
| 870 |
+
|
| 871 |
+
@property
|
| 872 |
def num_params(self):
|
| 873 |
return sum(p.numel() for p in self.parameters())
|
| 874 |
|
| 875 |
+
class RateConditionedController(nn.Module):
|
| 876 |
+
def __init__(self, input_dim, n_alloc, width=160, dropout=0.08):
|
| 877 |
+
super().__init__()
|
| 878 |
+
self.norm = nn.LayerNorm(input_dim + 1)
|
| 879 |
+
self.fc1 = nn.Linear(input_dim + 1, width)
|
| 880 |
+
self.fc2 = nn.Linear(width, width)
|
| 881 |
+
self.fc3 = nn.Linear(width, width)
|
| 882 |
+
self.drop = nn.Dropout(dropout)
|
| 883 |
+
self.head = nn.Linear(width, n_alloc)
|
| 884 |
|
| 885 |
+
def forward(self, x, lambda_norm):
|
| 886 |
+
if lambda_norm.ndim == 1:
|
| 887 |
+
lambda_norm = lambda_norm[:, None]
|
| 888 |
+
h = self.norm(torch.cat([x, lambda_norm], dim=-1))
|
| 889 |
+
h = F.gelu(self.fc1(h)); r = h
|
| 890 |
+
h = self.drop(F.gelu(self.fc2(h)))
|
| 891 |
+
h = F.gelu(self.fc3(h) + r)
|
| 892 |
+
return self.head(h)
|
| 893 |
+
|
| 894 |
+
def branch_scalar_features(x):
|
| 895 |
+
ax = x.abs()
|
| 896 |
+
df = (x[:, 1:] - x[:, :-1]).abs().mean(dim=(-2, -1))
|
| 897 |
+
dd = (torch.roll(x, -1, -1) - x).abs().mean(dim=(-2, -1))
|
| 898 |
+
lap = torch.zeros_like(x)
|
| 899 |
+
lap[:, 1:-1] = x[:, 2:] - 2*x[:, 1:-1] + x[:, :-2]
|
| 900 |
+
return torch.stack([
|
| 901 |
+
ax.mean(dim=(-2, -1)), ax.std(dim=(-2, -1)), ax.amax(dim=(-2, -1)),
|
| 902 |
+
df, dd, lap.abs().mean(dim=(-2, -1)),
|
| 903 |
+
], dim=-1)
|
| 904 |
+
|
| 905 |
+
def controller_features(batch, li, lb):
|
| 906 |
+
intra = branch_scalar_features(batch["intra_target"])
|
| 907 |
+
bg = branch_scalar_features(batch["bg_struct"])
|
| 908 |
+
prior = branch_scalar_features(batch["main_prior"])
|
| 909 |
+
gate = torch.stack([
|
| 910 |
+
batch["prior_gate"].mean(dim=(-2, -1)),
|
| 911 |
+
batch["prior_gate"].std(dim=(-2, -1)),
|
| 912 |
+
], dim=-1)
|
| 913 |
+
conf_i = torch.tanh(li).abs(); conf_b = torch.tanh(lb).abs()
|
| 914 |
+
prefix_points = [4, 8, 12, 16]
|
| 915 |
+
ci = torch.stack([conf_i[:, :p].mean(-1) for p in prefix_points], dim=-1)
|
| 916 |
+
cb = torch.stack([conf_b[:, :p].mean(-1) for p in prefix_points], dim=-1)
|
| 917 |
+
ratio = torch.stack([
|
| 918 |
+
intra[:, 0] / prior[:, 0].clamp_min(1e-5),
|
| 919 |
+
bg[:, 0] / prior[:, 0].clamp_min(1e-5),
|
| 920 |
+
], dim=-1)
|
| 921 |
+
return torch.cat([intra, bg, prior, gate, ci, cb, ratio], dim=-1)
|
| 922 |
+
|
| 923 |
+
|
| 924 |
+
class ProtocolRuntime:
|
| 925 |
+
"""Runtime view of the frozen C4-A/C4-B exact 30-byte packet protocol."""
|
| 926 |
+
|
| 927 |
+
def __init__(self, protocol: dict):
|
| 928 |
+
self.protocol = protocol
|
| 929 |
+
self.semantic_fields = list(protocol["semantic_fields"])
|
| 930 |
+
self.field_specs = list(protocol["field_specs"])
|
| 931 |
+
self.allocation_menu = [tuple(map(int, a)) for a in protocol["reduced_allocation_menu"]]
|
| 932 |
+
self.morphology_k = int(protocol["morphology_K"])
|
| 933 |
+
self.morphology_bits = int(protocol["morphology_bits_per_wave"])
|
| 934 |
+
self.packet_bytes = int(protocol["packet_bytes"])
|
| 935 |
+
self.payload_bytes = int(protocol["payload_bytes"])
|
| 936 |
+
self.crc_bytes = int(protocol["crc_bytes"])
|
| 937 |
+
self.semantic_profiles = {}
|
| 938 |
+
for key, bits in protocol["semantic_profiles"].items():
|
| 939 |
+
left, right = key.split("_")
|
| 940 |
+
n = int(left[1:])
|
| 941 |
+
aid = int(right[1:])
|
| 942 |
+
self.semantic_profiles[(n, aid)] = list(map(int, bits))
|
| 943 |
+
if self.packet_bytes != 30:
|
| 944 |
+
raise ValueError(f"Expected exact 30-byte packet protocol, got {self.packet_bytes}")
|
| 945 |
+
|
| 946 |
+
@staticmethod
|
| 947 |
+
def sem_internal_to_packet(sem9: torch.Tensor) -> torch.Tensor:
|
| 948 |
+
theta = torch.remainder(torch.atan2(sem9[..., 2], sem9[..., 3]), 2 * math.pi) / (2 * math.pi)
|
| 949 |
+
return torch.stack([
|
| 950 |
+
sem9[..., 0], sem9[..., 1], theta, sem9[..., 4], sem9[..., 5],
|
| 951 |
+
sem9[..., 6], sem9[..., 7], sem9[..., 8],
|
| 952 |
+
], dim=-1)
|
| 953 |
+
|
| 954 |
+
@staticmethod
|
| 955 |
+
def sem_packet_to_internal(p8: torch.Tensor) -> torch.Tensor:
|
| 956 |
+
theta = p8[..., 2] * (2 * math.pi)
|
| 957 |
+
return torch.stack([
|
| 958 |
+
p8[..., 0], p8[..., 1], torch.sin(theta), torch.cos(theta),
|
| 959 |
+
p8[..., 3], p8[..., 4], p8[..., 5], p8[..., 6], p8[..., 7],
|
| 960 |
+
], dim=-1)
|
| 961 |
+
|
| 962 |
+
@staticmethod
|
| 963 |
+
def _forward_compand(x: torch.Tensor, spec: dict) -> torch.Tensor:
|
| 964 |
+
kind = spec["kind"]
|
| 965 |
+
lo, hi = float(spec["lo"]), float(spec["hi"])
|
| 966 |
+
if kind == "circular":
|
| 967 |
+
return torch.remainder((x - lo) / (hi - lo), 1.0)
|
| 968 |
+
if kind == "linear":
|
| 969 |
+
return ((x - lo) / (hi - lo)).clamp(0, 1)
|
| 970 |
+
if kind == "log":
|
| 971 |
+
lx = torch.log(x.clamp_min(lo))
|
| 972 |
+
l0, l1 = math.log(lo), math.log(hi)
|
| 973 |
+
return ((lx - l0) / (l1 - l0)).clamp(0, 1)
|
| 974 |
+
if kind == "asinh":
|
| 975 |
+
s = float(spec.get("scale", 1.0))
|
| 976 |
+
a0, a1 = math.asinh(lo / s), math.asinh(hi / s)
|
| 977 |
+
return ((torch.asinh(x / s) - a0) / (a1 - a0)).clamp(0, 1)
|
| 978 |
+
raise ValueError(kind)
|
| 979 |
+
|
| 980 |
+
@staticmethod
|
| 981 |
+
def _inverse_compand(y: torch.Tensor, spec: dict) -> torch.Tensor:
|
| 982 |
+
kind = spec["kind"]
|
| 983 |
+
lo, hi = float(spec["lo"]), float(spec["hi"])
|
| 984 |
+
if kind in ("linear", "circular"):
|
| 985 |
+
x = lo + (hi - lo) * y
|
| 986 |
+
if kind == "circular":
|
| 987 |
+
x = lo + torch.remainder(x - lo, hi - lo)
|
| 988 |
+
return x
|
| 989 |
+
if kind == "log":
|
| 990 |
+
l0, l1 = math.log(lo), math.log(hi)
|
| 991 |
+
return torch.exp(l0 + (l1 - l0) * y)
|
| 992 |
+
if kind == "asinh":
|
| 993 |
+
s = float(spec.get("scale", 1.0))
|
| 994 |
+
a0, a1 = math.asinh(lo / s), math.asinh(hi / s)
|
| 995 |
+
return s * torch.sinh(a0 + (a1 - a0) * y)
|
| 996 |
+
raise ValueError(kind)
|
| 997 |
+
|
| 998 |
+
def packet_fields_to_unit(self, p8: torch.Tensor) -> torch.Tensor:
|
| 999 |
+
return torch.stack([
|
| 1000 |
+
self._forward_compand(p8[..., d], self.field_specs[d]) for d in range(8)
|
| 1001 |
+
], dim=-1)
|
| 1002 |
+
|
| 1003 |
+
def unit_to_packet_fields(self, y8: torch.Tensor) -> torch.Tensor:
|
| 1004 |
+
return torch.stack([
|
| 1005 |
+
self._inverse_compand(y8[..., d], self.field_specs[d]) for d in range(8)
|
| 1006 |
+
], dim=-1)
|
| 1007 |
+
|
| 1008 |
+
@staticmethod
|
| 1009 |
+
def project_unit_fields(y8: torch.Tensor) -> torch.Tensor:
|
| 1010 |
+
cols = []
|
| 1011 |
+
for d in range(8):
|
| 1012 |
+
v = y8[..., d]
|
| 1013 |
+
cols.append(torch.remainder(v, 1.0) if d == 2 else v.clamp(0, 1))
|
| 1014 |
+
return torch.stack(cols, dim=-1)
|
| 1015 |
+
|
| 1016 |
+
def profile_bits_tensor(self, counts: torch.Tensor, aids: torch.Tensor, device) -> torch.Tensor:
|
| 1017 |
+
rows = []
|
| 1018 |
+
for n, a in zip(counts.detach().cpu().tolist(), aids.detach().cpu().tolist()):
|
| 1019 |
+
rows.append(self.semantic_profiles[(max(1, int(n)), int(a))])
|
| 1020 |
+
return torch.tensor(rows, dtype=torch.long, device=device)
|
| 1021 |
+
|
| 1022 |
+
|
| 1023 |
+
class PacketSemanticQATAdapter(nn.Module):
|
| 1024 |
+
"""C4-B bounded packet-domain semantic pre-quantization adapter."""
|
| 1025 |
+
|
| 1026 |
+
def __init__(self, width: int = 96, delta_steps: float = 1.25, protocol: ProtocolRuntime | None = None):
|
| 1027 |
+
super().__init__()
|
| 1028 |
+
self.delta_steps = float(delta_steps)
|
| 1029 |
+
self.protocol = protocol
|
| 1030 |
+
self.net = nn.Sequential(
|
| 1031 |
+
nn.LayerNorm(21),
|
| 1032 |
+
nn.Linear(21, width), nn.GELU(),
|
| 1033 |
+
nn.Linear(width, width), nn.GELU(),
|
| 1034 |
+
nn.Linear(width, 8),
|
| 1035 |
+
)
|
| 1036 |
+
nn.init.zeros_(self.net[-1].weight)
|
| 1037 |
+
nn.init.zeros_(self.net[-1].bias)
|
| 1038 |
+
|
| 1039 |
+
def set_protocol(self, protocol: ProtocolRuntime) -> None:
|
| 1040 |
+
self.protocol = protocol
|
| 1041 |
+
|
| 1042 |
+
def _proto(self) -> ProtocolRuntime:
|
| 1043 |
+
if self.protocol is None:
|
| 1044 |
+
raise RuntimeError("PacketSemanticQATAdapter requires a ProtocolRuntime")
|
| 1045 |
+
return self.protocol
|
| 1046 |
+
|
| 1047 |
+
def adjust_unit(self, sem9, counts, aids, active):
|
| 1048 |
+
proto = self._proto()
|
| 1049 |
+
p8 = proto.sem_internal_to_packet(sem9)
|
| 1050 |
+
y = proto.packet_fields_to_unit(p8)
|
| 1051 |
+
bits = proto.profile_bits_tensor(counts, aids, sem9.device)
|
| 1052 |
+
levels = (torch.pow(2.0, bits.float()) - 1.0).clamp_min(1.0)
|
| 1053 |
+
step = 1.0 / levels
|
| 1054 |
+
bit_ctx = (bits.float() / 9.0)[:, None, :].expand(-1, sem9.shape[1], -1)
|
| 1055 |
+
cnt_ctx = (counts.float() / float(K_MAX))[:, None, None].expand(-1, sem9.shape[1], 1)
|
| 1056 |
+
aid_ctx = F.one_hot(aids.long(), num_classes=4).float()[:, None, :].expand(-1, sem9.shape[1], -1)
|
| 1057 |
+
x = torch.cat([y, bit_ctx, cnt_ctx, aid_ctx], dim=-1)
|
| 1058 |
+
raw = self.net(x)
|
| 1059 |
+
delta = self.delta_steps * step[:, None, :] * torch.tanh(raw)
|
| 1060 |
+
y_adj = proto.project_unit_fields(y + delta * active[..., None])
|
| 1061 |
+
return y, y_adj, step, bits
|
| 1062 |
+
|
| 1063 |
+
def fake_quantize(self, sem9, counts, aids, active):
|
| 1064 |
+
proto = self._proto()
|
| 1065 |
+
y, y_adj, step, bits = self.adjust_unit(sem9, counts, aids, active)
|
| 1066 |
+
levels = (torch.pow(2.0, bits.float()) - 1.0).clamp_min(1.0)[:, None, :]
|
| 1067 |
+
hard = torch.round(y_adj * levels) / levels
|
| 1068 |
+
yq = y_adj + (hard - y_adj).detach()
|
| 1069 |
+
semq = proto.sem_packet_to_internal(proto.unit_to_packet_fields(yq))
|
| 1070 |
+
return semq, dict(y=y, y_adj=y_adj, step=step, bits=bits, hard=hard)
|
| 1071 |
+
|
| 1072 |
+
@torch.no_grad()
|
| 1073 |
+
def hard_quantize(self, sem9, counts, aids, active):
|
| 1074 |
+
proto = self._proto()
|
| 1075 |
+
y, y_adj, step, bits = self.adjust_unit(sem9, counts, aids, active)
|
| 1076 |
+
levels = (torch.pow(2.0, bits.float()) - 1.0).clamp_min(1.0)[:, None, :]
|
| 1077 |
+
q = torch.round(y_adj * levels).long().clamp_min(0)
|
| 1078 |
+
hard = q.float() / levels
|
| 1079 |
+
semq = proto.sem_packet_to_internal(proto.unit_to_packet_fields(hard))
|
| 1080 |
+
return q, semq, dict(y=y, y_adj=y_adj, step=step, bits=bits, hard=hard)
|
| 1081 |
+
|
| 1082 |
+
@property
|
| 1083 |
+
def num_params(self):
|
| 1084 |
+
return sum(p.numel() for p in self.parameters())
|
| 1085 |
+
|
| 1086 |
+
|
| 1087 |
+
class PacketAwareController(nn.Module):
|
| 1088 |
+
def __init__(self,input_dim,width=160,dropout=0.08,n_alloc=4):
|
| 1089 |
+
super().__init__(); self.norm=nn.LayerNorm(input_dim+1)
|
| 1090 |
+
self.fc1=nn.Linear(input_dim+1,width); self.fc2=nn.Linear(width,width); self.fc3=nn.Linear(width,width)
|
| 1091 |
+
self.drop=nn.Dropout(dropout); self.head=nn.Linear(width,n_alloc)
|
| 1092 |
+
def forward(self,x,lambda_norm):
|
| 1093 |
+
if lambda_norm.ndim==1: lambda_norm=lambda_norm[:,None]
|
| 1094 |
+
h=self.norm(torch.cat([x,lambda_norm],dim=-1)); h=F.gelu(self.fc1(h)); r=h
|
| 1095 |
+
h=self.drop(F.gelu(self.fc2(h))); h=F.gelu(self.fc3(h)+r); return self.head(h)
|
| 1096 |
+
|
| 1097 |
+
|
| 1098 |
+
class ICWDSFrontEndCodec(nn.Module):
|
| 1099 |
+
"""
|
| 1100 |
+
Consolidated Stage-2 front-end feature codec.
|
| 1101 |
+
|
| 1102 |
+
Historical project naming keeps the archive name ``VAE.pt`` for compatibility,
|
| 1103 |
+
but the deployed object is a hybrid semantic/morphology/residual packet codec:
|
| 1104 |
+
C2 AC-SSVAE backbone + C3-D morphology selector + C3-F dual residual codec
|
| 1105 |
+
+ C4-B packet QAT adapter + packet-aware allocation controller.
|
| 1106 |
+
"""
|
| 1107 |
+
|
| 1108 |
+
def __init__(
|
| 1109 |
+
self,
|
| 1110 |
+
base_model: V5AwareStructuredSemanticVAE,
|
| 1111 |
+
morph_selector: PrototypeSelector,
|
| 1112 |
+
residual_codec: AdaptiveDualResidualCodec,
|
| 1113 |
+
qat_adapter: PacketSemanticQATAdapter,
|
| 1114 |
+
packet_controller: PacketAwareController,
|
| 1115 |
+
protocol_runtime: ProtocolRuntime,
|
| 1116 |
+
morph_centers: torch.Tensor,
|
| 1117 |
+
selector_x_mean: torch.Tensor,
|
| 1118 |
+
selector_x_std: torch.Tensor,
|
| 1119 |
+
feature_mean: torch.Tensor,
|
| 1120 |
+
feature_std: torch.Tensor,
|
| 1121 |
+
lambdas: list[float],
|
| 1122 |
+
):
|
| 1123 |
+
super().__init__()
|
| 1124 |
+
self.base_model = base_model
|
| 1125 |
+
self.morph_selector = morph_selector
|
| 1126 |
+
self.residual_codec = residual_codec
|
| 1127 |
+
self.qat_adapter = qat_adapter
|
| 1128 |
+
self.packet_controller = packet_controller
|
| 1129 |
+
self.protocol_runtime = protocol_runtime
|
| 1130 |
+
self.qat_adapter.set_protocol(protocol_runtime)
|
| 1131 |
+
self.register_buffer("morph_centers", morph_centers.float())
|
| 1132 |
+
self.register_buffer("selector_x_mean", selector_x_mean.float())
|
| 1133 |
+
self.register_buffer("selector_x_std", selector_x_std.float().clamp_min(1e-5))
|
| 1134 |
+
self.register_buffer("feature_mean", feature_mean.float())
|
| 1135 |
+
self.register_buffer("feature_std", feature_std.float().clamp_min(1e-5))
|
| 1136 |
+
self.lambdas = [float(x) for x in lambdas]
|
| 1137 |
+
|
| 1138 |
+
@torch.no_grad()
|
| 1139 |
+
def decode_semantic_prior(self, sem9, morph_idx, active):
|
| 1140 |
+
z = self.morph_centers.to(sem9.device)[morph_idx.long()]
|
| 1141 |
+
dec = self.base_model.decode_from_codes(sem9, z, exist_prob=active)
|
| 1142 |
+
return dec["part_hat"].sum(1).clamp(0, 1)
|
| 1143 |
+
|
| 1144 |
+
def quantize_semantics(self, sem9, counts, allocation_ids, active, hard=True):
|
| 1145 |
+
if hard:
|
| 1146 |
+
return self.qat_adapter.hard_quantize(sem9, counts, allocation_ids, active)
|
| 1147 |
+
return self.qat_adapter.fake_quantize(sem9, counts, allocation_ids, active)
|
| 1148 |
+
|
| 1149 |
+
@torch.no_grad()
|
| 1150 |
+
def choose_allocation(self, features: torch.Tensor, mode: str = "quality") -> torch.Tensor:
|
| 1151 |
+
if mode not in {"quality", "comm"}:
|
| 1152 |
+
raise ValueError("mode must be 'quality' or 'comm'")
|
| 1153 |
+
lam = self.lambdas[0] if mode == "quality" else self.lambdas[-1]
|
| 1154 |
+
max_lam = max(self.lambdas)
|
| 1155 |
+
x = (features - self.feature_mean) / self.feature_std
|
| 1156 |
+
ln = x.new_full((len(x), 1), lam / max(max_lam, 1e-12))
|
| 1157 |
+
return self.packet_controller(x, ln).argmax(-1)
|
| 1158 |
+
|
| 1159 |
+
@classmethod
|
| 1160 |
+
def from_archive(cls, archive_path: str, map_location: str | torch.device = "cpu"):
|
| 1161 |
+
bundle = torch.load(archive_path, map_location=map_location, weights_only=False)
|
| 1162 |
+
if bundle.get("format") != "ICWDS_VAE_ARCHIVE_V1":
|
| 1163 |
+
raise ValueError("Not an ICWDS consolidated VAE.pt archive")
|
| 1164 |
+
|
| 1165 |
+
c2 = bundle["c2_checkpoint"]
|
| 1166 |
+
c3d = bundle["c3d_package"]
|
| 1167 |
+
c3f = bundle["c3f_package"]
|
| 1168 |
+
c4b = bundle["c4b_package"]
|
| 1169 |
+
|
| 1170 |
+
cfg_dict = c2.get("config", {})
|
| 1171 |
+
allowed = set(V5AwareACSSVAEConfig.__dataclass_fields__)
|
| 1172 |
+
cfg = V5AwareACSSVAEConfig(**{k: v for k, v in cfg_dict.items() if k in allowed})
|
| 1173 |
+
base_model = V5AwareStructuredSemanticVAE(cfg)
|
| 1174 |
+
base_model.load_state_dict(c2["model"], strict=True)
|
| 1175 |
+
|
| 1176 |
+
morph_k = int(c4b.get("protocol", {}).get("morphology_K", 4))
|
| 1177 |
+
sel_item = c3d["selectors"][morph_k]
|
| 1178 |
+
morph_selector = PrototypeSelector(
|
| 1179 |
+
input_dim=int(c3d["input_dim"]),
|
| 1180 |
+
n_codes=morph_k,
|
| 1181 |
+
width=int(sel_item.get("width", 192)),
|
| 1182 |
+
dropout=float(sel_item.get("dropout", 0.10)),
|
| 1183 |
+
)
|
| 1184 |
+
morph_selector.load_state_dict(sel_item["state_dict"], strict=True)
|
| 1185 |
+
|
| 1186 |
+
c3f_ctrl = c3f.get("control", {})
|
| 1187 |
+
residual_codec = AdaptiveDualResidualCodec(
|
| 1188 |
+
max_bits=int(c3f_ctrl.get("MAX_BITS_PER_BRANCH", 24)),
|
| 1189 |
+
enc_width=int(c3f_ctrl.get("ENC_WIDTH", 32)),
|
| 1190 |
+
dec_width=int(c3f_ctrl.get("DEC_WIDTH", 32)),
|
| 1191 |
+
)
|
| 1192 |
+
residual_codec.load_state_dict(c3f["codec"], strict=True)
|
| 1193 |
+
|
| 1194 |
+
proto = ProtocolRuntime(c4b["protocol"])
|
| 1195 |
+
qa = c4b["qat_adapter_arch"]
|
| 1196 |
+
qat_adapter = PacketSemanticQATAdapter(
|
| 1197 |
+
width=int(qa["width"]),
|
| 1198 |
+
delta_steps=float(qa["delta_steps"]),
|
| 1199 |
+
protocol=proto,
|
| 1200 |
+
)
|
| 1201 |
+
qat_adapter.load_state_dict(c4b["qat_adapter"], strict=True)
|
| 1202 |
+
|
| 1203 |
+
pa = c4b["packet_controller_arch"]
|
| 1204 |
+
packet_controller = PacketAwareController(
|
| 1205 |
+
input_dim=int(pa["input_dim"]),
|
| 1206 |
+
width=int(pa["width"]),
|
| 1207 |
+
dropout=float(pa["dropout"]),
|
| 1208 |
+
n_alloc=int(pa["n_alloc"]),
|
| 1209 |
+
)
|
| 1210 |
+
packet_controller.load_state_dict(c4b["packet_controller"], strict=True)
|
| 1211 |
+
|
| 1212 |
+
model = cls(
|
| 1213 |
+
base_model=base_model,
|
| 1214 |
+
morph_selector=morph_selector,
|
| 1215 |
+
residual_codec=residual_codec,
|
| 1216 |
+
qat_adapter=qat_adapter,
|
| 1217 |
+
packet_controller=packet_controller,
|
| 1218 |
+
protocol_runtime=proto,
|
| 1219 |
+
morph_centers=sel_item["centers"],
|
| 1220 |
+
selector_x_mean=c3d["x_mean"],
|
| 1221 |
+
selector_x_std=c3d["x_std"],
|
| 1222 |
+
feature_mean=c4b["feature_mean"],
|
| 1223 |
+
feature_std=c4b["feature_std"],
|
| 1224 |
+
lambdas=list(c4b["lambdas"]),
|
| 1225 |
+
)
|
| 1226 |
+
return model
|
| 1227 |
+
|
| 1228 |
+
|
| 1229 |
+
def build_vae_archive_from_files(
|
| 1230 |
+
c2_checkpoint_path: str,
|
| 1231 |
+
c3d_package_path: str,
|
| 1232 |
+
c3f_package_path: str,
|
| 1233 |
+
c4b_package_path: str,
|
| 1234 |
+
output_path: str = "VAE.pt",
|
| 1235 |
+
) -> str:
|
| 1236 |
+
"""Consolidate the four validated Stage-2 artifacts into one portable VAE.pt."""
|
| 1237 |
+
def load(path):
|
| 1238 |
+
return torch.load(path, map_location="cpu", weights_only=False)
|
| 1239 |
+
|
| 1240 |
+
bundle = {
|
| 1241 |
+
"format": "ICWDS_VAE_ARCHIVE_V1",
|
| 1242 |
+
"description": "C2 AC-SSVAE + C3-D morphology + C3-F residual codec + C4-B QAT/controller",
|
| 1243 |
+
"c2_checkpoint": load(c2_checkpoint_path),
|
| 1244 |
+
"c3d_package": load(c3d_package_path),
|
| 1245 |
+
"c3f_package": load(c3f_package_path),
|
| 1246 |
+
"c4b_package": load(c4b_package_path),
|
| 1247 |
+
}
|
| 1248 |
+
torch.save(bundle, output_path)
|
| 1249 |
+
return output_path
|
| 1250 |
+
|
| 1251 |
+
|
| 1252 |
+
def build_vae_archive_from_hf(
|
| 1253 |
+
output_path: str = "VAE.pt",
|
| 1254 |
+
repo_id: str = "wuff-mann/Wave_compress_system",
|
| 1255 |
+
token: str | None = None,
|
| 1256 |
+
) -> str:
|
| 1257 |
+
"""Download the four frozen artifacts from HF and build one self-contained VAE.pt."""
|
| 1258 |
+
try:
|
| 1259 |
+
from huggingface_hub import hf_hub_download
|
| 1260 |
+
except ImportError as exc:
|
| 1261 |
+
raise RuntimeError("Install huggingface_hub first") from exc
|
| 1262 |
+
|
| 1263 |
+
paths = {
|
| 1264 |
+
"c2": "WaveSemanticHybridCodec/V5_ACSSVAE_C2_CausalProtectedReal/checkpoints/C2_consolidate/complete.pt",
|
| 1265 |
+
"c3d": "WaveSemanticHybridCodec/V5_ACSSVAE_C3D_ReconstructionAwarePrototypeSelector/C3D_prototype_selector_package.pt",
|
| 1266 |
+
"c3f": "WaveSemanticHybridCodec/V5_ACSSVAE_C3F_AdaptiveDualResidualCodec/C3F_adaptive_dual_residual_codec_package.pt",
|
| 1267 |
+
"c4b": "WaveSemanticHybridCodec/V5_ACSSVAE_C4B_PacketAwareQAT_Controller/C4B_packet_qat_controller_package.pt",
|
| 1268 |
+
}
|
| 1269 |
+
local = {
|
| 1270 |
+
k: hf_hub_download(repo_id=repo_id, filename=v, repo_type="model", token=token)
|
| 1271 |
+
for k, v in paths.items()
|
| 1272 |
+
}
|
| 1273 |
+
return build_vae_archive_from_files(local["c2"], local["c3d"], local["c3f"], local["c4b"], output_path)
|
| 1274 |
+
|
| 1275 |
+
|
| 1276 |
+
def load_vae_archive(path: str = "VAE.pt", map_location: str | torch.device = "cpu") -> ICWDSFrontEndCodec:
|
| 1277 |
+
return ICWDSFrontEndCodec.from_archive(path, map_location=map_location)
|
| 1278 |
+
|
| 1279 |
+
|
| 1280 |
+
__all__ = [
|
| 1281 |
+
"V5AwareACSSVAEConfig",
|
| 1282 |
+
"V5AwareStructuredSemanticVAE",
|
| 1283 |
+
"PrototypeSelector",
|
| 1284 |
+
"AdaptiveDualResidualCodec",
|
| 1285 |
+
"RateConditionedController",
|
| 1286 |
+
"PacketSemanticQATAdapter",
|
| 1287 |
+
"PacketAwareController",
|
| 1288 |
+
"ProtocolRuntime",
|
| 1289 |
+
"ICWDSFrontEndCodec",
|
| 1290 |
+
"build_vae_archive_from_files",
|
| 1291 |
+
"build_vae_archive_from_hf",
|
| 1292 |
+
"load_vae_archive",
|
| 1293 |
+
]
|
| 1294 |
+
|
| 1295 |
+
|
| 1296 |
+
if __name__ == "__main__":
|
| 1297 |
+
import argparse
|
| 1298 |
+
parser = argparse.ArgumentParser(description="Build consolidated ICWDS VAE.pt archive")
|
| 1299 |
+
parser.add_argument("--build-from-hf", action="store_true")
|
| 1300 |
+
parser.add_argument("--output", default="VAE.pt")
|
| 1301 |
+
parser.add_argument("--repo", default="wuff-mann/Wave_compress_system")
|
| 1302 |
+
parser.add_argument("--token", default=None)
|
| 1303 |
+
args = parser.parse_args()
|
| 1304 |
+
if args.build_from_hf:
|
| 1305 |
+
out = build_vae_archive_from_hf(args.output, args.repo, args.token)
|
| 1306 |
+
print(out)
|