""" PixelModel v2 - the weights ARE the image. model.png stores every parameter of the network as pixel values. v2 changes vs v1: - Wider trunk and decoder: 200,259 parameters rather than 23,747, while keeping the same prompt encoder, coordinate features, 64x64 native resolution, and 16-bit PNG codec. - The larger capacity is intended for the same ~20K COCO training rows. v1 retained these changes from v0: - Coordinate-conditioned decoder (CPPN-style): parameters no longer scale with output resolution. v0 spent 196,608 of its 202,752 params on the output head; v1 has 23,747 params total and renders at ANY resolution (native 64x64 for eval). - Hashed character-trigram + word prompt embedding (deterministic, 0 params) instead of v0's char-sum, so different prompts actually get different, partially compositional embeddings. - Biases everywhere (v0 had none). - 16-bit weight codec: R = high byte, G = low byte, B = reserved. v0 wasted G and stored weights at 8 bits; v1's quantization error is ~6e-5 per weight. model.png is 160x149 px - the model is a thumbnail. Architecture: prompt -> hashed trigram/word embedding (64) -> T1 (256x64)+b tanh -> T2 (128x256)+b tanh -> latent z (128) per pixel: concat(z, fourier(x,y) (18)) -> D1 (320x146)+b tanh -> D2 (320x320)+b tanh -> D3 (3x320)+b sigmoid -> RGB """ import numpy as np import torch from PIL import Image # -- config ------------------------------------------------------------------- EMB_DIM = 64 # prompt embedding size (hashed, no learned params) TRUNK_HIDDEN = 256 LATENT = 128 COORD_FEATS = 18 # x, y + sin/cos at freqs (1,2,4,8) for each axis DEC_HIDDEN = 320 NATIVE_RES = 64 # native eval resolution (decoder works at any res) FREQS = (1.0, 2.0, 4.0, 8.0) # every parameter tensor, in canonical flattening order PARAM_SPECS = [ ("T1", (TRUNK_HIDDEN, EMB_DIM)), ("b1", (TRUNK_HIDDEN,)), ("T2", (LATENT, TRUNK_HIDDEN)), ("b2", (LATENT,)), ("D1", (DEC_HIDDEN, LATENT + COORD_FEATS)), ("bd1", (DEC_HIDDEN,)), ("D2", (DEC_HIDDEN, DEC_HIDDEN)), ("bd2", (DEC_HIDDEN,)), ("D3", (3, DEC_HIDDEN)), ("bd3", (3,)), ] N_PARAMS = sum(int(np.prod(s)) for _, s in PARAM_SPECS) # 200,259 # model.png geometry: one weight per pixel, 16-bit (R=high, G=low, B=reserved) MODEL_W = 256 MODEL_H = -(-N_PARAMS // MODEL_W) # 783 WMAX = 2.0 # weights live in [-2, 2] # -- prompt embedding (deterministic, zero parameters) ------------------------ def _fnv1a(data: bytes) -> int: h = 0x811C9DC5 for byte in data: h ^= byte h = (h * 0x01000193) & 0xFFFFFFFF return h def prompt_to_embedding(prompt: str) -> torch.Tensor: """Hashed bag of character trigrams + words -> EMB_DIM vector, L2-normed.""" text = "".join(c if c.isalnum() or c == " " else " " for c in prompt.lower()) text = " ".join(text.split()) vec = np.zeros(EMB_DIM, dtype=np.float32) padded = f" {text} " for i in range(len(padded) - 2): h = _fnv1a(padded[i:i + 3].encode("utf-8")) sign = 1.0 if (h >> 16) & 1 else -1.0 vec[h % EMB_DIM] += sign for word in text.split(): h = _fnv1a(b"w:" + word.encode("utf-8")) sign = 1.0 if (h >> 16) & 1 else -1.0 vec[h % EMB_DIM] += 2.0 * sign # words weigh more than trigrams norm = np.linalg.norm(vec) if norm > 0: vec /= norm return torch.from_numpy(vec) def prompts_to_embeddings(prompts) -> torch.Tensor: return torch.stack([prompt_to_embedding(p) for p in prompts]) # -- coordinate features ------------------------------------------------------ _coord_cache = {} def coord_features(res: int) -> torch.Tensor: """(res*res, COORD_FEATS) fourier features of the pixel grid, cached.""" if res not in _coord_cache: axis = torch.linspace(-1.0, 1.0, res) yy, xx = torch.meshgrid(axis, axis, indexing="ij") x = xx.reshape(-1) y = yy.reshape(-1) feats = [x, y] for f in FREQS: feats += [torch.sin(f * torch.pi * x), torch.cos(f * torch.pi * x), torch.sin(f * torch.pi * y), torch.cos(f * torch.pi * y)] _coord_cache[res] = torch.stack(feats, dim=1) # (res*res, 18) return _coord_cache[res] # -- forward ------------------------------------------------------------------ def encode_prompt(weights: dict, emb: torch.Tensor) -> torch.Tensor: """emb (B, EMB_DIM) -> latent z (B, LATENT).""" z = torch.tanh(emb @ weights["T1"].T + weights["b1"]) z = torch.tanh(z @ weights["T2"].T + weights["b2"]) return z def decode_pixels(weights: dict, z: torch.Tensor, feats: torch.Tensor) -> torch.Tensor: """z (B, LATENT), feats (P, COORD_FEATS) -> RGB (B, P, 3) in [0, 1].""" B, P = z.shape[0], feats.shape[0] inp = torch.cat([z.unsqueeze(1).expand(B, P, LATENT), feats.unsqueeze(0).expand(B, P, COORD_FEATS)], dim=2) h = torch.tanh(inp @ weights["D1"].T + weights["bd1"]) h = torch.tanh(h @ weights["D2"].T + weights["bd2"]) return torch.sigmoid(h @ weights["D3"].T + weights["bd3"]) def forward(weights: dict, prompts, res: int = NATIVE_RES) -> torch.Tensor: """prompts: str or list of str -> (B, res, res, 3) float in [0, 1].""" if isinstance(prompts, str): prompts = [prompts] emb = prompts_to_embeddings(prompts) z = encode_prompt(weights, emb) rgb = decode_pixels(weights, z, coord_features(res)) return rgb.reshape(len(prompts), res, res, 3) # -- 16-bit PNG weight codec -------------------------------------------------- def weights_to_pixels(weights: dict) -> np.ndarray: """weight dict -> (MODEL_H, MODEL_W, 3) uint8. R=high byte, G=low, B=0.""" flat = torch.cat([weights[n].detach().reshape(-1) for n, _ in PARAM_SPECS]) q = ((flat.clamp(-WMAX, WMAX) / WMAX + 1.0) / 2.0 * 65535.0).round() q = q.to(torch.int64).cpu().numpy() q = np.pad(q, (0, MODEL_W * MODEL_H - N_PARAMS)) img = np.zeros((MODEL_H * MODEL_W, 3), dtype=np.uint8) img[:, 0] = q >> 8 img[:, 1] = q & 0xFF return img.reshape(MODEL_H, MODEL_W, 3) def pixels_to_weights(arr: np.ndarray) -> dict: """(MODEL_H, MODEL_W, 3) uint8 -> weight dict.""" flat = arr.reshape(-1, 3).astype(np.int64) q = (flat[:, 0] << 8) | flat[:, 1] vals = (q.astype(np.float32) / 65535.0 * 2.0 - 1.0) * WMAX vals = torch.from_numpy(vals[:N_PARAMS]) weights, i = {}, 0 for name, shape in PARAM_SPECS: n = int(np.prod(shape)) weights[name] = vals[i:i + n].reshape(shape).clone() i += n return weights def save_model(weights: dict, path: str): Image.fromarray(weights_to_pixels(weights), mode="RGB").save(path) def load_model(path: str) -> dict: img = Image.open(path).convert("RGB") arr = np.array(img, dtype=np.uint8) assert arr.shape == (MODEL_H, MODEL_W, 3), \ f"expected {MODEL_H}x{MODEL_W} model.png, got {arr.shape[0]}x{arr.shape[1]}" return pixels_to_weights(arr) def init_weights(seed: int = 0) -> dict: g = torch.Generator().manual_seed(seed) weights = {} for name, shape in PARAM_SPECS: if len(shape) == 1: weights[name] = torch.zeros(shape) else: fan_in = shape[1] scale = 1.0 / fan_in ** 0.5 if name == "D3": scale *= 0.1 # start near grey, not saturated weights[name] = torch.randn(shape, generator=g) * scale return weights if __name__ == "__main__": print(f"params: {N_PARAMS} model.png: {MODEL_W}x{MODEL_H} px") for name, shape in PARAM_SPECS: print(f" {name:>4} {str(tuple(shape)):>12} {int(np.prod(shape)):>6}")