add txt2img tab + infer_txt2img endpoint (SD 1.5)
Browse files- app.py +131 -27
- pixel_cursor/rigging.py +127 -0
- requirements.txt +1 -0
- studio/backends/hf_space.py +1 -1
- studio/cli.py +26 -0
- studio/rigging/__init__.py +91 -0
- studio/rigging/deform.py +171 -0
- studio/rigging/joints.py +166 -0
- studio/rigging/overlay.py +86 -0
- studio/rigging/parts.py +287 -0
- studio/rigging/rigid_piece.py +124 -0
- studio/rigging/segment.py +70 -0
- studio/rigging/sidecar.py +139 -0
- studio/rigging/skin.py +84 -0
- studio/spritesheet.py +78 -0
app.py
CHANGED
|
@@ -2,15 +2,19 @@
|
|
| 2 |
|
| 3 |
Push this whole repo to a HF Space (sdk: gradio). The Space's Python build will
|
| 4 |
install `requirements.txt` (torch + diffusers + transformers + gradio + spaces)
|
| 5 |
-
and call
|
| 6 |
-
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
Locally this file is not imported — `studio` CLI uses `studio.backends.hf_space`
|
| 10 |
to call this Space remotely via gradio_client.
|
| 11 |
"""
|
| 12 |
from __future__ import annotations
|
| 13 |
import io
|
|
|
|
| 14 |
|
| 15 |
import gradio as gr
|
| 16 |
import numpy as np
|
|
@@ -23,6 +27,7 @@ from studio.backends.animatediff import AnimateDiffAdapter, MOTION_LORA_MAP
|
|
| 23 |
|
| 24 |
|
| 25 |
_adapter: AnimateDiffAdapter | None = None
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
def _get_adapter() -> AnimateDiffAdapter:
|
|
@@ -33,6 +38,31 @@ def _get_adapter() -> AnimateDiffAdapter:
|
|
| 33 |
return _adapter
|
| 34 |
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
@spaces.GPU(duration=90)
|
| 37 |
def infer(
|
| 38 |
image: np.ndarray,
|
|
@@ -67,35 +97,109 @@ def infer(
|
|
| 67 |
return out.name
|
| 68 |
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
with gr.Blocks(title="Venture-Studio") as demo:
|
| 71 |
gr.Markdown(
|
| 72 |
"# Venture-Studio · Pixel-Cursor Animation\n"
|
| 73 |
-
"Single image → 24fps animation
|
| 74 |
-
"Running on Hugging Face ZeroGPU (free
|
| 75 |
)
|
| 76 |
-
with gr.
|
| 77 |
-
with gr.
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
)
|
| 82 |
-
with gr.Accordion("Advanced", open=False):
|
| 83 |
-
num_frames = gr.Slider(8, 24, value=16, step=2, label="num_frames")
|
| 84 |
-
steps = gr.Slider(10, 50, value=25, step=1, label="num_inference_steps")
|
| 85 |
-
guidance = gr.Slider(1.0, 15.0, value=7.5, step=0.5, label="guidance_scale")
|
| 86 |
-
prompt = gr.Textbox(value="high quality, detailed", label="prompt")
|
| 87 |
-
neg = gr.Textbox(value="bad quality, blurry", label="negative_prompt")
|
| 88 |
-
seed = gr.Number(value=42, precision=0, label="seed")
|
| 89 |
-
run = gr.Button("Generate", variant="primary")
|
| 90 |
-
with gr.Column():
|
| 91 |
-
video_out = gr.Video(label="Output", autoplay=True, loop=True)
|
| 92 |
-
|
| 93 |
-
run.click(
|
| 94 |
-
infer,
|
| 95 |
-
inputs=[image_in, preset, num_frames, steps, guidance, prompt, neg, seed],
|
| 96 |
-
outputs=video_out,
|
| 97 |
-
api_name="infer",
|
| 98 |
-
)
|
| 99 |
|
| 100 |
|
| 101 |
if __name__ == "__main__":
|
|
|
|
| 2 |
|
| 3 |
Push this whole repo to a HF Space (sdk: gradio). The Space's Python build will
|
| 4 |
install `requirements.txt` (torch + diffusers + transformers + gradio + spaces)
|
| 5 |
+
and call:
|
| 6 |
+
- `app.py:infer` — single image → motion (AnimateDiff MotionLoRA)
|
| 7 |
+
- `app.py:infer_txt2img` — prompt → 512×512 sprite (SD 1.5)
|
| 8 |
+
|
| 9 |
+
`@spaces.GPU` allocates an A10G only for the duration of the call (ZeroGPU
|
| 10 |
+
model), so the Space is free for the maintainer and shared fairly across users.
|
| 11 |
|
| 12 |
Locally this file is not imported — `studio` CLI uses `studio.backends.hf_space`
|
| 13 |
to call this Space remotely via gradio_client.
|
| 14 |
"""
|
| 15 |
from __future__ import annotations
|
| 16 |
import io
|
| 17 |
+
import tempfile
|
| 18 |
|
| 19 |
import gradio as gr
|
| 20 |
import numpy as np
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
_adapter: AnimateDiffAdapter | None = None
|
| 30 |
+
_sd_pipe = None
|
| 31 |
|
| 32 |
|
| 33 |
def _get_adapter() -> AnimateDiffAdapter:
|
|
|
|
| 38 |
return _adapter
|
| 39 |
|
| 40 |
|
| 41 |
+
def _get_sd_pipe():
|
| 42 |
+
"""Lazily load SD 1.5 txt2img pipeline (runs inside @spaces.GPU context).
|
| 43 |
+
|
| 44 |
+
Cached at module scope so warm calls skip re-loading. Cold start is ~30s
|
| 45 |
+
including weight download on first call ever (cached in persistent storage
|
| 46 |
+
for subsequent cold starts).
|
| 47 |
+
"""
|
| 48 |
+
global _sd_pipe
|
| 49 |
+
if _sd_pipe is not None:
|
| 50 |
+
return _sd_pipe
|
| 51 |
+
import torch
|
| 52 |
+
from diffusers import AutoPipelineForText2Image, DPMSolverMultistepScheduler
|
| 53 |
+
pipe = AutoPipelineForText2Image.from_pretrained(
|
| 54 |
+
"runwayml/stable-diffusion-v1-5",
|
| 55 |
+
torch_dtype=torch.float16,
|
| 56 |
+
safety_checker=None,
|
| 57 |
+
requires_safety_checker=False,
|
| 58 |
+
)
|
| 59 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
|
| 60 |
+
pipe = pipe.to("cuda")
|
| 61 |
+
pipe.set_progress_bar_config(disable=True)
|
| 62 |
+
_sd_pipe = pipe
|
| 63 |
+
return pipe
|
| 64 |
+
|
| 65 |
+
|
| 66 |
@spaces.GPU(duration=90)
|
| 67 |
def infer(
|
| 68 |
image: np.ndarray,
|
|
|
|
| 97 |
return out.name
|
| 98 |
|
| 99 |
|
| 100 |
+
@spaces.GPU(duration=45)
|
| 101 |
+
def infer_txt2img(
|
| 102 |
+
prompt: str,
|
| 103 |
+
negative_prompt: str,
|
| 104 |
+
num_inference_steps: int,
|
| 105 |
+
guidance_scale: float,
|
| 106 |
+
height: int,
|
| 107 |
+
width: int,
|
| 108 |
+
seed: int,
|
| 109 |
+
) -> str:
|
| 110 |
+
"""Generate a single sprite from a text prompt. Returns path to a PNG.
|
| 111 |
+
|
| 112 |
+
Defaults tuned for pixel-art sprites: 512×512, 25 steps, guidance 7.5.
|
| 113 |
+
Caller downscales to target res (256×256 is the Dicer sprite size).
|
| 114 |
+
"""
|
| 115 |
+
import torch
|
| 116 |
+
|
| 117 |
+
pipe = _get_sd_pipe()
|
| 118 |
+
g = torch.Generator(device="cuda").manual_seed(int(seed))
|
| 119 |
+
out = pipe(
|
| 120 |
+
prompt=prompt,
|
| 121 |
+
negative_prompt=negative_prompt,
|
| 122 |
+
num_inference_steps=int(num_inference_steps),
|
| 123 |
+
guidance_scale=float(guidance_scale),
|
| 124 |
+
height=int(height),
|
| 125 |
+
width=int(width),
|
| 126 |
+
generator=g,
|
| 127 |
+
)
|
| 128 |
+
image = out.images[0]
|
| 129 |
+
f = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
|
| 130 |
+
image.save(f.name)
|
| 131 |
+
return f.name
|
| 132 |
+
|
| 133 |
+
|
| 134 |
with gr.Blocks(title="Venture-Studio") as demo:
|
| 135 |
gr.Markdown(
|
| 136 |
"# Venture-Studio · Pixel-Cursor Animation\n"
|
| 137 |
+
"Single image → 24fps animation, or text prompt → sprite, via PixelCursor.\n"
|
| 138 |
+
"Running on Hugging Face ZeroGPU (free A10G)."
|
| 139 |
)
|
| 140 |
+
with gr.Tabs():
|
| 141 |
+
with gr.Tab("Motion"):
|
| 142 |
+
with gr.Row():
|
| 143 |
+
with gr.Column():
|
| 144 |
+
image_in = gr.Image(label="Source image", type="numpy", height=384)
|
| 145 |
+
preset = gr.Dropdown(
|
| 146 |
+
choices=sorted(MOTION_LORA_MAP), value="zoom_in",
|
| 147 |
+
label="MotionLoRA preset",
|
| 148 |
+
)
|
| 149 |
+
with gr.Accordion("Advanced", open=False):
|
| 150 |
+
num_frames = gr.Slider(8, 24, value=16, step=2, label="num_frames")
|
| 151 |
+
steps = gr.Slider(10, 50, value=25, step=1, label="num_inference_steps")
|
| 152 |
+
guidance = gr.Slider(1.0, 15.0, value=7.5, step=0.5, label="guidance_scale")
|
| 153 |
+
prompt = gr.Textbox(value="high quality, detailed", label="prompt")
|
| 154 |
+
neg = gr.Textbox(value="bad quality, blurry", label="negative_prompt")
|
| 155 |
+
seed = gr.Number(value=42, precision=0, label="seed")
|
| 156 |
+
run = gr.Button("Generate", variant="primary")
|
| 157 |
+
with gr.Column():
|
| 158 |
+
video_out = gr.Video(label="Output", autoplay=True, loop=True)
|
| 159 |
+
|
| 160 |
+
run.click(
|
| 161 |
+
infer,
|
| 162 |
+
inputs=[image_in, preset, num_frames, steps, guidance, prompt, neg, seed],
|
| 163 |
+
outputs=video_out,
|
| 164 |
+
api_name="infer",
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
with gr.Tab("Sprite gen (txt2img)"):
|
| 168 |
+
with gr.Row():
|
| 169 |
+
with gr.Column():
|
| 170 |
+
t2i_prompt = gr.Textbox(
|
| 171 |
+
value=(
|
| 172 |
+
"pixel art, 16-bit fantasy goblin warrior, empty hands, "
|
| 173 |
+
"no weapon, standing pose, full body, centered, "
|
| 174 |
+
"white background, sharp pixels, blocky, retro game sprite"
|
| 175 |
+
),
|
| 176 |
+
lines=3, label="prompt",
|
| 177 |
+
)
|
| 178 |
+
t2i_neg = gr.Textbox(
|
| 179 |
+
value=(
|
| 180 |
+
"sword, weapon, dagger, axe, staff, blurry, soft, "
|
| 181 |
+
"anti-aliasing, smooth, photorealistic, 3d render, "
|
| 182 |
+
"extra limbs, distorted"
|
| 183 |
+
),
|
| 184 |
+
lines=2, label="negative_prompt",
|
| 185 |
+
)
|
| 186 |
+
with gr.Accordion("Advanced", open=False):
|
| 187 |
+
t2i_steps = gr.Slider(10, 50, value=25, step=1, label="num_inference_steps")
|
| 188 |
+
t2i_guidance = gr.Slider(1.0, 15.0, value=7.5, step=0.5, label="guidance_scale")
|
| 189 |
+
t2i_height = gr.Slider(256, 768, value=512, step=64, label="height")
|
| 190 |
+
t2i_width = gr.Slider(256, 768, value=512, step=64, label="width")
|
| 191 |
+
t2i_seed = gr.Number(value=0, precision=0, label="seed (0 = random)")
|
| 192 |
+
t2i_run = gr.Button("Generate sprite", variant="primary")
|
| 193 |
+
with gr.Column():
|
| 194 |
+
t2i_out = gr.Image(label="Generated sprite", height=512)
|
| 195 |
+
|
| 196 |
+
t2i_run.click(
|
| 197 |
+
infer_txt2img,
|
| 198 |
+
inputs=[t2i_prompt, t2i_neg, t2i_steps, t2i_guidance,
|
| 199 |
+
t2i_height, t2i_width, t2i_seed],
|
| 200 |
+
outputs=t2i_out,
|
| 201 |
+
api_name="infer_txt2img",
|
| 202 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
|
| 204 |
|
| 205 |
if __name__ == "__main__":
|
pixel_cursor/rigging.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""pixel_cursor.rigging — 14-joint humanoid skeleton primitives.
|
| 2 |
+
|
| 3 |
+
Session 1 (Phase 3) scope: data type, canonical joint layout, parent-linkage.
|
| 4 |
+
Detection / segmentation / overlay live in studio.rigging.
|
| 5 |
+
|
| 6 |
+
The skeleton is image-space (pixel coordinates, (y, x) order). Coordinates are
|
| 7 |
+
floats so sub-pixel joint positions (from distance-transform peak interpolation)
|
| 8 |
+
survive without rounding loss.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Final, Tuple
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
JOINT_NAMES: Final[Tuple[str, ...]] = (
|
| 19 |
+
"head", # 0
|
| 20 |
+
"neck", # 1 — root (parent = -1)
|
| 21 |
+
"l_shoulder", # 2
|
| 22 |
+
"r_shoulder", # 3
|
| 23 |
+
"l_elbow", # 4
|
| 24 |
+
"r_elbow", # 5
|
| 25 |
+
"l_wrist", # 6
|
| 26 |
+
"r_wrist", # 7
|
| 27 |
+
"l_hip", # 8
|
| 28 |
+
"r_hip", # 9
|
| 29 |
+
"l_knee", # 10
|
| 30 |
+
"r_knee", # 11
|
| 31 |
+
"l_ankle", # 12
|
| 32 |
+
"r_ankle", # 13
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
NUM_JOINTS: Final[int] = len(JOINT_NAMES)
|
| 36 |
+
|
| 37 |
+
PARENTS: Final[Tuple[int, ...]] = (
|
| 38 |
+
1, # head -> neck
|
| 39 |
+
-1, # neck -> root
|
| 40 |
+
1, # l_shoulder -> neck
|
| 41 |
+
1, # r_shoulder -> neck
|
| 42 |
+
2, # l_elbow -> l_shoulder
|
| 43 |
+
3, # r_elbow -> r_shoulder
|
| 44 |
+
4, # l_wrist -> l_elbow
|
| 45 |
+
5, # r_wrist -> r_elbow
|
| 46 |
+
1, # l_hip -> neck
|
| 47 |
+
1, # r_hip -> neck
|
| 48 |
+
8, # l_knee -> l_hip
|
| 49 |
+
9, # r_knee -> r_hip
|
| 50 |
+
10, # l_ankle -> l_knee
|
| 51 |
+
11, # r_ankle -> r_knee
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
JOINT_INDEX: Final[dict[str, int]] = {name: i for i, name in enumerate(JOINT_NAMES)}
|
| 55 |
+
|
| 56 |
+
BONES: Final[Tuple[Tuple[int, int], ...]] = tuple(
|
| 57 |
+
(PARENTS[i], i) for i in range(NUM_JOINTS) if PARENTS[i] >= 0
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True, eq=False, repr=False)
|
| 62 |
+
class Skeleton:
|
| 63 |
+
positions: np.ndarray # shape (14, 2), float32, (y, x) per joint
|
| 64 |
+
confidence: np.ndarray # shape (14,), float32, in [0, 1]
|
| 65 |
+
image_shape: Tuple[int, int] # (H, W) of the source mask
|
| 66 |
+
|
| 67 |
+
def __post_init__(self) -> None:
|
| 68 |
+
if self.positions.shape != (NUM_JOINTS, 2):
|
| 69 |
+
raise ValueError(
|
| 70 |
+
f"positions must be shape ({NUM_JOINTS}, 2), got {self.positions.shape}"
|
| 71 |
+
)
|
| 72 |
+
if self.confidence.shape != (NUM_JOINTS,):
|
| 73 |
+
raise ValueError(
|
| 74 |
+
f"confidence must be shape ({NUM_JOINTS},), got {self.confidence.shape}"
|
| 75 |
+
)
|
| 76 |
+
if self.positions.dtype != np.float32:
|
| 77 |
+
object.__setattr__(self, "positions", self.positions.astype(np.float32))
|
| 78 |
+
if self.confidence.dtype != np.float32:
|
| 79 |
+
object.__setattr__(self, "confidence", self.confidence.astype(np.float32))
|
| 80 |
+
# Lock buffers so the frozen contract survives array views.
|
| 81 |
+
self.positions.setflags(write=False)
|
| 82 |
+
self.confidence.setflags(write=False)
|
| 83 |
+
|
| 84 |
+
def __repr__(self) -> str:
|
| 85 |
+
return (
|
| 86 |
+
f"Skeleton(image_shape={self.image_shape}, "
|
| 87 |
+
f"joints={NUM_JOINTS}, "
|
| 88 |
+
f"mean_confidence={float(self.confidence.mean()):.2f})"
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
@classmethod
|
| 92 |
+
def from_positions(
|
| 93 |
+
cls,
|
| 94 |
+
positions: np.ndarray,
|
| 95 |
+
image_shape: Tuple[int, int],
|
| 96 |
+
confidence: np.ndarray | None = None,
|
| 97 |
+
) -> "Skeleton":
|
| 98 |
+
positions = np.asarray(positions, dtype=np.float32)
|
| 99 |
+
if confidence is None:
|
| 100 |
+
confidence = np.ones(NUM_JOINTS, dtype=np.float32)
|
| 101 |
+
else:
|
| 102 |
+
confidence = np.asarray(confidence, dtype=np.float32)
|
| 103 |
+
return cls(positions=positions, confidence=confidence, image_shape=image_shape)
|
| 104 |
+
|
| 105 |
+
def joint(self, name: str) -> Tuple[float, float]:
|
| 106 |
+
idx = JOINT_INDEX[name]
|
| 107 |
+
return float(self.positions[idx, 0]), float(self.positions[idx, 1])
|
| 108 |
+
|
| 109 |
+
def to_dict(self) -> dict[str, dict[str, float]]:
|
| 110 |
+
return {
|
| 111 |
+
name: {
|
| 112 |
+
"y": float(self.positions[i, 0]),
|
| 113 |
+
"x": float(self.positions[i, 1]),
|
| 114 |
+
"confidence": float(self.confidence[i]),
|
| 115 |
+
}
|
| 116 |
+
for i, name in enumerate(JOINT_NAMES)
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
__all__ = [
|
| 121 |
+
"JOINT_NAMES",
|
| 122 |
+
"NUM_JOINTS",
|
| 123 |
+
"PARENTS",
|
| 124 |
+
"JOINT_INDEX",
|
| 125 |
+
"BONES",
|
| 126 |
+
"Skeleton",
|
| 127 |
+
]
|
requirements.txt
CHANGED
|
@@ -4,6 +4,7 @@
|
|
| 4 |
# do NOT pin it here or builds can conflict.
|
| 5 |
numpy>=1.26
|
| 6 |
Pillow>=10.0
|
|
|
|
| 7 |
imageio>=2.34
|
| 8 |
imageio-ffmpeg>=0.5
|
| 9 |
torch>=2.2
|
|
|
|
| 4 |
# do NOT pin it here or builds can conflict.
|
| 5 |
numpy>=1.26
|
| 6 |
Pillow>=10.0
|
| 7 |
+
scipy>=1.11
|
| 8 |
imageio>=2.34
|
| 9 |
imageio-ffmpeg>=0.5
|
| 10 |
torch>=2.2
|
studio/backends/hf_space.py
CHANGED
|
@@ -130,7 +130,7 @@ class HFSpaceAdapter:
|
|
| 130 |
raise ImportError(INSTALL_HINT) from e
|
| 131 |
|
| 132 |
token = _resolve_hf_token(self.hf_token)
|
| 133 |
-
client = Client(self.space_id,
|
| 134 |
|
| 135 |
_, preset, _ = cursor.identity_lock.source.split(":", 2)
|
| 136 |
ms = motion_spec or {}
|
|
|
|
| 130 |
raise ImportError(INSTALL_HINT) from e
|
| 131 |
|
| 132 |
token = _resolve_hf_token(self.hf_token)
|
| 133 |
+
client = Client(self.space_id, token=token)
|
| 134 |
|
| 135 |
_, preset, _ = cursor.identity_lock.source.split(":", 2)
|
| 136 |
ms = motion_spec or {}
|
studio/cli.py
CHANGED
|
@@ -45,6 +45,22 @@ def cmd_export(args: argparse.Namespace) -> int:
|
|
| 45 |
return 0
|
| 46 |
|
| 47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
def cmd_motion(args: argparse.Namespace) -> int:
|
| 49 |
from .backends.animatediff import AnimateDiffAdapter, MOTION_LORA_MAP
|
| 50 |
from .backends.hf_space import HFSpaceAdapter
|
|
@@ -141,6 +157,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|
| 141 |
)
|
| 142 |
p_export.set_defaults(func=cmd_export)
|
| 143 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
p_motion = sub.add_parser(
|
| 145 |
"motion",
|
| 146 |
help="Generate motion on a single image via AnimateDiff MotionLoRA preset",
|
|
|
|
| 45 |
return 0
|
| 46 |
|
| 47 |
|
| 48 |
+
def cmd_sheet(args: argparse.Namespace) -> int:
|
| 49 |
+
from .spritesheet import write_sprite_sheet, sheet_metadata
|
| 50 |
+
|
| 51 |
+
stack = load_framestack_npz(args.frames)
|
| 52 |
+
out = write_sprite_sheet(stack, args.out, cols=args.cols)
|
| 53 |
+
meta = sheet_metadata(stack, cols=args.cols)
|
| 54 |
+
print(f"sheet -> {out}")
|
| 55 |
+
print(json.dumps(meta, indent=2))
|
| 56 |
+
if args.manifest:
|
| 57 |
+
manifest_path = args.manifest
|
| 58 |
+
with open(manifest_path, "w") as f:
|
| 59 |
+
json.dump(meta, f, indent=2)
|
| 60 |
+
print(f"manifest -> {manifest_path}")
|
| 61 |
+
return 0
|
| 62 |
+
|
| 63 |
+
|
| 64 |
def cmd_motion(args: argparse.Namespace) -> int:
|
| 65 |
from .backends.animatediff import AnimateDiffAdapter, MOTION_LORA_MAP
|
| 66 |
from .backends.hf_space import HFSpaceAdapter
|
|
|
|
| 157 |
)
|
| 158 |
p_export.set_defaults(func=cmd_export)
|
| 159 |
|
| 160 |
+
p_sheet = sub.add_parser(
|
| 161 |
+
"sheet",
|
| 162 |
+
help="Pack a FrameStack .npz into a sprite-sheet PNG (Babylon SpriteManager friendly)",
|
| 163 |
+
)
|
| 164 |
+
p_sheet.add_argument("frames", help="input .npz from `studio compile` or `studio motion`")
|
| 165 |
+
p_sheet.add_argument("-o", "--out", required=True, help="output .png path")
|
| 166 |
+
p_sheet.add_argument("--cols", type=int, default=4, help="frames per row in the grid (default 4)")
|
| 167 |
+
p_sheet.add_argument("--manifest", help="optional .json manifest path (cell size, fps, loop ms)")
|
| 168 |
+
p_sheet.set_defaults(func=cmd_sheet)
|
| 169 |
+
|
| 170 |
p_motion = sub.add_parser(
|
| 171 |
"motion",
|
| 172 |
help="Generate motion on a single image via AnimateDiff MotionLoRA preset",
|
studio/rigging/__init__.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging — auto-rigging foundation (Phase 3).
|
| 2 |
+
|
| 3 |
+
Picked stack (see ../../PHASE_3_RESEARCH.md):
|
| 4 |
+
segment.flood_fill_segment — flood-fill from corners + alpha fast path
|
| 5 |
+
joints.detect_joints — distance-transform + mask-topology hybrid
|
| 6 |
+
overlay.render_overlay — numpy + PIL debug visualisation
|
| 7 |
+
sidecar.load_sidecar/save_sidecar — operator-annotation JSON I/O
|
| 8 |
+
|
| 9 |
+
Top-level entry point:
|
| 10 |
+
rig_sprite(image_path) — tries sidecar first, falls back to auto-detect
|
| 11 |
+
|
| 12 |
+
Mesh skinning and pose deformation belong to Session 3 and are not implemented here.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from typing import Tuple
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
from PIL import Image
|
| 21 |
+
|
| 22 |
+
from pixel_cursor.rigging import Skeleton
|
| 23 |
+
|
| 24 |
+
from .deform import deform_sprite, forward_kinematics
|
| 25 |
+
from .joints import detect_joints
|
| 26 |
+
from .overlay import render_overlay
|
| 27 |
+
from .parts import (
|
| 28 |
+
CANONICAL_HUMANOID_PIECES,
|
| 29 |
+
PartsMask,
|
| 30 |
+
Piece,
|
| 31 |
+
canonical_humanoid_pieces,
|
| 32 |
+
load_parts,
|
| 33 |
+
parts_json_path_for,
|
| 34 |
+
parts_png_path_for,
|
| 35 |
+
parts_preview_path_for,
|
| 36 |
+
save_parts,
|
| 37 |
+
save_preview,
|
| 38 |
+
)
|
| 39 |
+
from .rigid_piece import deform_sprite_rigid
|
| 40 |
+
from .segment import flood_fill_segment
|
| 41 |
+
from .sidecar import (
|
| 42 |
+
SCHEMA_VERSION,
|
| 43 |
+
load_sidecar,
|
| 44 |
+
save_sidecar,
|
| 45 |
+
sidecar_path_for,
|
| 46 |
+
sidecar_template,
|
| 47 |
+
)
|
| 48 |
+
from .skin import compute_skin_weights
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def rig_sprite(image_path: Path | str) -> Tuple[Skeleton, np.ndarray, str]:
|
| 52 |
+
"""Produce a 14-joint skeleton for a sprite on disk.
|
| 53 |
+
|
| 54 |
+
Strategy: sidecar JSON first (operator-authoritative), auto-detect fallback.
|
| 55 |
+
|
| 56 |
+
Returns (skeleton, mask, source) where source ∈ {"sidecar", "auto"}.
|
| 57 |
+
"""
|
| 58 |
+
path = Path(image_path)
|
| 59 |
+
image = np.asarray(Image.open(path))
|
| 60 |
+
mask = flood_fill_segment(image)
|
| 61 |
+
sidecar = load_sidecar(path, image_shape=mask.shape)
|
| 62 |
+
if sidecar is not None:
|
| 63 |
+
return sidecar, mask, "sidecar"
|
| 64 |
+
return detect_joints(mask), mask, "auto"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
__all__ = [
|
| 68 |
+
"flood_fill_segment",
|
| 69 |
+
"detect_joints",
|
| 70 |
+
"render_overlay",
|
| 71 |
+
"rig_sprite",
|
| 72 |
+
"load_sidecar",
|
| 73 |
+
"save_sidecar",
|
| 74 |
+
"sidecar_path_for",
|
| 75 |
+
"sidecar_template",
|
| 76 |
+
"SCHEMA_VERSION",
|
| 77 |
+
"compute_skin_weights",
|
| 78 |
+
"forward_kinematics",
|
| 79 |
+
"deform_sprite",
|
| 80 |
+
"CANONICAL_HUMANOID_PIECES",
|
| 81 |
+
"PartsMask",
|
| 82 |
+
"Piece",
|
| 83 |
+
"canonical_humanoid_pieces",
|
| 84 |
+
"load_parts",
|
| 85 |
+
"parts_json_path_for",
|
| 86 |
+
"parts_png_path_for",
|
| 87 |
+
"parts_preview_path_for",
|
| 88 |
+
"save_parts",
|
| 89 |
+
"save_preview",
|
| 90 |
+
"deform_sprite_rigid",
|
| 91 |
+
]
|
studio/rigging/deform.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.deform — forward kinematics + Linear Blend Skinning warp.
|
| 2 |
+
|
| 3 |
+
Pose representation: a dict mapping joint_name → local rotation (radians). Local
|
| 4 |
+
rotations propagate down the tree to produce a world rotation per joint; each
|
| 5 |
+
joint's deformed position is its parent's deformed position plus the rest offset
|
| 6 |
+
rotated by the parent's world rotation.
|
| 7 |
+
|
| 8 |
+
Pixel warping is LBS forward-splat: each foreground pixel computes its blended
|
| 9 |
+
output position as Σ_b weight_b · T_b(pixel), then bilinearly splats its color
|
| 10 |
+
into the output buffer. Subject preservation is structural: every output pixel
|
| 11 |
+
is a weighted combination of source pixels — no pixel is invented.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from typing import Mapping, Tuple
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
from pixel_cursor.rigging import BONES, JOINT_INDEX, NUM_JOINTS, PARENTS, Skeleton
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _eval_order() -> Tuple[int, ...]:
|
| 23 |
+
root_idx = PARENTS.index(-1)
|
| 24 |
+
order = [root_idx]
|
| 25 |
+
queue = [root_idx]
|
| 26 |
+
seen = {root_idx}
|
| 27 |
+
while queue:
|
| 28 |
+
parent = queue.pop(0)
|
| 29 |
+
for child, parent_idx in enumerate(PARENTS):
|
| 30 |
+
if parent_idx == parent and child not in seen:
|
| 31 |
+
order.append(child)
|
| 32 |
+
seen.add(child)
|
| 33 |
+
queue.append(child)
|
| 34 |
+
return tuple(order)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
_EVAL_ORDER: Tuple[int, ...] = _eval_order()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def forward_kinematics(
|
| 41 |
+
rest: Skeleton,
|
| 42 |
+
local_rotations: Mapping[str, float] | None = None,
|
| 43 |
+
) -> Tuple[np.ndarray, np.ndarray]:
|
| 44 |
+
"""Run FK from a dict of local rotations.
|
| 45 |
+
|
| 46 |
+
Returns (deformed_positions shape (14, 2), world_rotations shape (14,)).
|
| 47 |
+
"""
|
| 48 |
+
local = np.zeros(NUM_JOINTS, dtype=np.float32)
|
| 49 |
+
if local_rotations:
|
| 50 |
+
for name, angle in local_rotations.items():
|
| 51 |
+
local[JOINT_INDEX[name]] = float(angle)
|
| 52 |
+
|
| 53 |
+
deformed = rest.positions.copy().astype(np.float32)
|
| 54 |
+
world_rot = np.zeros(NUM_JOINTS, dtype=np.float32)
|
| 55 |
+
|
| 56 |
+
root_idx = PARENTS.index(-1)
|
| 57 |
+
world_rot[root_idx] = local[root_idx]
|
| 58 |
+
|
| 59 |
+
for joint in _EVAL_ORDER:
|
| 60 |
+
if joint == root_idx:
|
| 61 |
+
continue
|
| 62 |
+
parent = PARENTS[joint]
|
| 63 |
+
world_rot[joint] = world_rot[parent] + local[joint]
|
| 64 |
+
rest_offset = rest.positions[joint] - rest.positions[parent]
|
| 65 |
+
c, s = np.cos(world_rot[parent]), np.sin(world_rot[parent])
|
| 66 |
+
rotated_y = c * rest_offset[0] - s * rest_offset[1]
|
| 67 |
+
rotated_x = s * rest_offset[0] + c * rest_offset[1]
|
| 68 |
+
deformed[joint, 0] = deformed[parent, 0] + rotated_y
|
| 69 |
+
deformed[joint, 1] = deformed[parent, 1] + rotated_x
|
| 70 |
+
|
| 71 |
+
return deformed.astype(np.float32), world_rot
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def deform_sprite(
|
| 75 |
+
image: np.ndarray,
|
| 76 |
+
mask: np.ndarray,
|
| 77 |
+
rest: Skeleton,
|
| 78 |
+
weights: np.ndarray,
|
| 79 |
+
*,
|
| 80 |
+
local_rotations: Mapping[str, float] | None = None,
|
| 81 |
+
) -> Tuple[np.ndarray, np.ndarray]:
|
| 82 |
+
"""Apply LBS warp to a sprite. Returns (deformed_rgba (H, W, 4), deformed_mask (H, W) bool).
|
| 83 |
+
|
| 84 |
+
Algorithm:
|
| 85 |
+
1. FK produces deformed joint positions + per-joint world rotations.
|
| 86 |
+
2. For each foreground source pixel p:
|
| 87 |
+
T_b(p) = deformed[parent(b)] + R(world_rot[parent(b)]) · (p - rest[parent(b)])
|
| 88 |
+
out_pos = Σ_b weights[p, b] · T_b(p)
|
| 89 |
+
3. Bilinearly splat source RGBA into the output accumulator at out_pos.
|
| 90 |
+
4. Normalise accumulator by accumulated weight per output pixel.
|
| 91 |
+
"""
|
| 92 |
+
if mask.ndim != 2:
|
| 93 |
+
raise ValueError(f"mask must be 2D, got {mask.shape}")
|
| 94 |
+
H, W = mask.shape
|
| 95 |
+
|
| 96 |
+
deformed_pos, world_rot = forward_kinematics(rest, local_rotations)
|
| 97 |
+
|
| 98 |
+
if image.ndim == 2:
|
| 99 |
+
rgba_in = np.stack(
|
| 100 |
+
[image, image, image, np.full_like(image, 255)], axis=-1
|
| 101 |
+
)
|
| 102 |
+
elif image.shape[-1] == 3:
|
| 103 |
+
rgba_in = np.concatenate(
|
| 104 |
+
[image, np.full(image.shape[:2] + (1,), 255, dtype=np.uint8)], axis=-1
|
| 105 |
+
)
|
| 106 |
+
elif image.shape[-1] == 4:
|
| 107 |
+
rgba_in = image
|
| 108 |
+
else:
|
| 109 |
+
raise ValueError(f"unsupported image shape {image.shape}")
|
| 110 |
+
|
| 111 |
+
mask_bool = mask.astype(bool)
|
| 112 |
+
ys, xs = np.where(mask_bool)
|
| 113 |
+
if ys.size == 0:
|
| 114 |
+
return np.zeros((H, W, 4), dtype=np.uint8), np.zeros((H, W), dtype=bool)
|
| 115 |
+
|
| 116 |
+
src_coords = np.stack([ys, xs], axis=-1).astype(np.float32)
|
| 117 |
+
src_colors = rgba_in[ys, xs].astype(np.float32)
|
| 118 |
+
src_weights = weights[ys, xs] # (N_fg, N_BONES)
|
| 119 |
+
|
| 120 |
+
out_pos = np.zeros_like(src_coords)
|
| 121 |
+
for b, (parent_idx, _child_idx) in enumerate(BONES):
|
| 122 |
+
rot = float(world_rot[parent_idx])
|
| 123 |
+
c, s = np.cos(rot), np.sin(rot)
|
| 124 |
+
offset = src_coords - rest.positions[parent_idx]
|
| 125 |
+
ry = c * offset[:, 0] - s * offset[:, 1]
|
| 126 |
+
rx = s * offset[:, 0] + c * offset[:, 1]
|
| 127 |
+
rotated = np.stack([ry, rx], axis=-1)
|
| 128 |
+
per_bone_out = deformed_pos[parent_idx] + rotated
|
| 129 |
+
out_pos += src_weights[:, b : b + 1] * per_bone_out
|
| 130 |
+
|
| 131 |
+
accum = np.zeros((H, W, 4), dtype=np.float32)
|
| 132 |
+
wsum = np.zeros((H, W), dtype=np.float32)
|
| 133 |
+
|
| 134 |
+
oy = out_pos[:, 0]
|
| 135 |
+
ox = out_pos[:, 1]
|
| 136 |
+
oy_int = np.floor(oy).astype(np.int32)
|
| 137 |
+
ox_int = np.floor(ox).astype(np.int32)
|
| 138 |
+
fy = oy - oy_int
|
| 139 |
+
fx = ox - ox_int
|
| 140 |
+
|
| 141 |
+
for dy in (0, 1):
|
| 142 |
+
for dx in (0, 1):
|
| 143 |
+
wy = (1.0 - fy) if dy == 0 else fy
|
| 144 |
+
wx = (1.0 - fx) if dx == 0 else fx
|
| 145 |
+
wgt = wy * wx
|
| 146 |
+
yy = oy_int + dy
|
| 147 |
+
xx = ox_int + dx
|
| 148 |
+
valid = (yy >= 0) & (yy < H) & (xx >= 0) & (xx < W) & (wgt > 0)
|
| 149 |
+
if not valid.any():
|
| 150 |
+
continue
|
| 151 |
+
yy_v = yy[valid]
|
| 152 |
+
xx_v = xx[valid]
|
| 153 |
+
w_v = wgt[valid]
|
| 154 |
+
for ch in range(4):
|
| 155 |
+
np.add.at(accum[..., ch], (yy_v, xx_v), w_v * src_colors[valid, ch])
|
| 156 |
+
np.add.at(wsum, (yy_v, xx_v), w_v)
|
| 157 |
+
|
| 158 |
+
deformed_rgba = np.zeros((H, W, 4), dtype=np.uint8)
|
| 159 |
+
has_color = wsum > 0
|
| 160 |
+
np.divide(
|
| 161 |
+
accum,
|
| 162 |
+
np.maximum(wsum[..., None], 1e-12),
|
| 163 |
+
out=accum,
|
| 164 |
+
where=has_color[..., None],
|
| 165 |
+
)
|
| 166 |
+
deformed_rgba[has_color] = np.clip(accum[has_color], 0, 255).astype(np.uint8)
|
| 167 |
+
|
| 168 |
+
return deformed_rgba, has_color
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
__all__ = ["forward_kinematics", "deform_sprite"]
|
studio/rigging/joints.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.joints — distance-transform + mask-topology joint detector.
|
| 2 |
+
|
| 3 |
+
Strategy: for each of 8 anatomical Y-fractions (head/neck/shoulder/elbow/wrist/
|
| 4 |
+
hip/knee/ankle), scan that row of the foreground mask, identify horizontal runs,
|
| 5 |
+
and map runs to joint indices. Distance-transform refines run centers to limb
|
| 6 |
+
midlines (avoids the "joint sits on the silhouette edge" failure mode).
|
| 7 |
+
|
| 8 |
+
Synthetic humanoids drawn with these same anatomical fractions verify the
|
| 9 |
+
detector within 8 px at every joint. Real cartoon sprites will need Session 2.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from typing import List, Optional, Tuple
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
from scipy.ndimage import distance_transform_edt
|
| 17 |
+
|
| 18 |
+
from pixel_cursor.rigging import NUM_JOINTS, Skeleton
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
ANATOMY_Y_FRACTIONS: dict[str, float] = {
|
| 22 |
+
"head": 0.079, # head center; mask top is head_top, not head_center
|
| 23 |
+
"neck": 0.158,
|
| 24 |
+
"shoulder": 0.189,
|
| 25 |
+
"elbow": 0.333,
|
| 26 |
+
"wrist": 0.482,
|
| 27 |
+
"hip": 0.588,
|
| 28 |
+
"knee": 0.789,
|
| 29 |
+
"ankle": 0.991,
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _runs_in_row(row: np.ndarray) -> List[Tuple[int, int]]:
|
| 34 |
+
if not row.any():
|
| 35 |
+
return []
|
| 36 |
+
padded = np.concatenate([[False], row, [False]])
|
| 37 |
+
diff = np.diff(padded.astype(np.int8))
|
| 38 |
+
starts = np.where(diff == 1)[0]
|
| 39 |
+
ends = np.where(diff == -1)[0] - 1
|
| 40 |
+
return list(zip(starts.tolist(), ends.tolist()))
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _refine_x_via_dt(dt_row: np.ndarray, x_start: int, x_end: int) -> float:
|
| 44 |
+
"""Within a horizontal run, return the X with maximum distance-transform value."""
|
| 45 |
+
segment = dt_row[x_start : x_end + 1]
|
| 46 |
+
best = int(np.argmax(segment))
|
| 47 |
+
return float(x_start + best)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _pick_lr_runs(
|
| 51 |
+
runs: List[Tuple[int, int]],
|
| 52 |
+
) -> Optional[Tuple[Tuple[int, int], Tuple[int, int]]]:
|
| 53 |
+
if len(runs) < 2:
|
| 54 |
+
return None
|
| 55 |
+
if len(runs) == 2:
|
| 56 |
+
a, b = sorted(runs, key=lambda r: r[0])
|
| 57 |
+
return a, b
|
| 58 |
+
sorted_runs = sorted(runs, key=lambda r: r[0])
|
| 59 |
+
return sorted_runs[0], sorted_runs[-1]
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def detect_joints(mask: np.ndarray) -> Skeleton:
|
| 63 |
+
if mask.ndim != 2:
|
| 64 |
+
raise ValueError(f"mask must be 2D, got shape {mask.shape}")
|
| 65 |
+
mask_bool = mask.astype(bool)
|
| 66 |
+
H, W = mask_bool.shape
|
| 67 |
+
|
| 68 |
+
ys, _ = np.where(mask_bool)
|
| 69 |
+
if ys.size == 0:
|
| 70 |
+
return Skeleton(
|
| 71 |
+
positions=np.zeros((NUM_JOINTS, 2), dtype=np.float32),
|
| 72 |
+
confidence=np.zeros(NUM_JOINTS, dtype=np.float32),
|
| 73 |
+
image_shape=(H, W),
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
y_min, y_max = int(ys.min()), int(ys.max())
|
| 77 |
+
bbox_h = y_max - y_min + 1
|
| 78 |
+
dt = distance_transform_edt(mask_bool).astype(np.float32)
|
| 79 |
+
|
| 80 |
+
positions = np.zeros((NUM_JOINTS, 2), dtype=np.float32)
|
| 81 |
+
confidence = np.ones(NUM_JOINTS, dtype=np.float32)
|
| 82 |
+
|
| 83 |
+
def anatomy_y(name: str) -> int:
|
| 84 |
+
return int(np.clip(y_min + ANATOMY_Y_FRACTIONS[name] * bbox_h, y_min, y_max))
|
| 85 |
+
|
| 86 |
+
def single_x(y: int) -> Tuple[float, float]:
|
| 87 |
+
runs = _runs_in_row(mask_bool[y])
|
| 88 |
+
if not runs:
|
| 89 |
+
return float(W) / 2.0, 0.0
|
| 90 |
+
s, e = max(runs, key=lambda r: r[1] - r[0])
|
| 91 |
+
return _refine_x_via_dt(dt[y], s, e), 1.0
|
| 92 |
+
|
| 93 |
+
def lr_x(y: int) -> Optional[Tuple[float, float]]:
|
| 94 |
+
runs = _runs_in_row(mask_bool[y])
|
| 95 |
+
pair = _pick_lr_runs(runs)
|
| 96 |
+
if pair is None:
|
| 97 |
+
return None
|
| 98 |
+
(ls, le), (rs, re) = pair
|
| 99 |
+
return _refine_x_via_dt(dt[y], ls, le), _refine_x_via_dt(dt[y], rs, re)
|
| 100 |
+
|
| 101 |
+
head_y = anatomy_y("head")
|
| 102 |
+
hx, hconf = single_x(head_y)
|
| 103 |
+
positions[0] = (head_y, hx)
|
| 104 |
+
confidence[0] = hconf
|
| 105 |
+
|
| 106 |
+
neck_y = anatomy_y("neck")
|
| 107 |
+
nx, nconf = single_x(neck_y)
|
| 108 |
+
positions[1] = (neck_y, nx)
|
| 109 |
+
confidence[1] = nconf
|
| 110 |
+
|
| 111 |
+
shoulder_y = anatomy_y("shoulder")
|
| 112 |
+
lr = lr_x(shoulder_y)
|
| 113 |
+
if lr is not None:
|
| 114 |
+
positions[2] = (shoulder_y, lr[0])
|
| 115 |
+
positions[3] = (shoulder_y, lr[1])
|
| 116 |
+
else:
|
| 117 |
+
runs = _runs_in_row(mask_bool[shoulder_y])
|
| 118 |
+
if runs:
|
| 119 |
+
s, e = max(runs, key=lambda r: r[1] - r[0])
|
| 120 |
+
positions[2] = (shoulder_y, s)
|
| 121 |
+
positions[3] = (shoulder_y, e)
|
| 122 |
+
confidence[2] = confidence[3] = 0.5
|
| 123 |
+
else:
|
| 124 |
+
confidence[2] = confidence[3] = 0.0
|
| 125 |
+
|
| 126 |
+
for joint_l, joint_r, key in ((4, 5, "elbow"), (6, 7, "wrist")):
|
| 127 |
+
y = anatomy_y(key)
|
| 128 |
+
lr = lr_x(y)
|
| 129 |
+
if lr is not None:
|
| 130 |
+
positions[joint_l] = (y, lr[0])
|
| 131 |
+
positions[joint_r] = (y, lr[1])
|
| 132 |
+
else:
|
| 133 |
+
positions[joint_l] = (y, positions[2, 1])
|
| 134 |
+
positions[joint_r] = (y, positions[3, 1])
|
| 135 |
+
confidence[joint_l] = confidence[joint_r] = 0.5
|
| 136 |
+
|
| 137 |
+
hip_y = anatomy_y("hip")
|
| 138 |
+
lr = lr_x(hip_y)
|
| 139 |
+
if lr is not None:
|
| 140 |
+
positions[8] = (hip_y, lr[0])
|
| 141 |
+
positions[9] = (hip_y, lr[1])
|
| 142 |
+
else:
|
| 143 |
+
runs = _runs_in_row(mask_bool[hip_y])
|
| 144 |
+
if runs:
|
| 145 |
+
s, e = max(runs, key=lambda r: r[1] - r[0])
|
| 146 |
+
positions[8] = (hip_y, s)
|
| 147 |
+
positions[9] = (hip_y, e)
|
| 148 |
+
confidence[8] = confidence[9] = 0.5
|
| 149 |
+
else:
|
| 150 |
+
confidence[8] = confidence[9] = 0.0
|
| 151 |
+
|
| 152 |
+
for joint_l, joint_r, key in ((10, 11, "knee"), (12, 13, "ankle")):
|
| 153 |
+
y = anatomy_y(key)
|
| 154 |
+
lr = lr_x(y)
|
| 155 |
+
if lr is not None:
|
| 156 |
+
positions[joint_l] = (y, lr[0])
|
| 157 |
+
positions[joint_r] = (y, lr[1])
|
| 158 |
+
else:
|
| 159 |
+
positions[joint_l] = (y, positions[8, 1])
|
| 160 |
+
positions[joint_r] = (y, positions[9, 1])
|
| 161 |
+
confidence[joint_l] = confidence[joint_r] = 0.5
|
| 162 |
+
|
| 163 |
+
return Skeleton(positions=positions, confidence=confidence, image_shape=(H, W))
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
__all__ = ["detect_joints", "ANATOMY_Y_FRACTIONS"]
|
studio/rigging/overlay.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.overlay — numpy + PIL debug overlay renderer.
|
| 2 |
+
|
| 3 |
+
Composites: source image → mask-edge ring → bones (cyan) → joints (red dots).
|
| 4 |
+
Output is a PIL.Image in RGBA, ready to save as PNG.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
from PIL import Image, ImageDraw
|
| 10 |
+
from scipy.ndimage import binary_erosion
|
| 11 |
+
|
| 12 |
+
from pixel_cursor.rigging import BONES, JOINT_NAMES, Skeleton
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
JOINT_COLOR = (255, 80, 80, 255)
|
| 16 |
+
JOINT_OUTLINE = (0, 0, 0, 255)
|
| 17 |
+
BONE_COLOR = (60, 200, 255, 255)
|
| 18 |
+
MASK_EDGE_COLOR = (255, 255, 0, 200)
|
| 19 |
+
JOINT_RADIUS = 4
|
| 20 |
+
BONE_WIDTH = 2
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def render_overlay(
|
| 24 |
+
image: np.ndarray,
|
| 25 |
+
mask: np.ndarray,
|
| 26 |
+
skeleton: Skeleton,
|
| 27 |
+
*,
|
| 28 |
+
show_labels: bool = False,
|
| 29 |
+
) -> Image.Image:
|
| 30 |
+
if image.ndim == 2:
|
| 31 |
+
rgba = np.stack(
|
| 32 |
+
[image, image, image, np.full_like(image, 255)],
|
| 33 |
+
axis=-1,
|
| 34 |
+
)
|
| 35 |
+
elif image.shape[-1] == 3:
|
| 36 |
+
rgba = np.concatenate(
|
| 37 |
+
[image, np.full(image.shape[:2] + (1,), 255, dtype=np.uint8)],
|
| 38 |
+
axis=-1,
|
| 39 |
+
)
|
| 40 |
+
elif image.shape[-1] == 4:
|
| 41 |
+
rgba = image.copy()
|
| 42 |
+
else:
|
| 43 |
+
raise ValueError(f"unsupported image shape {image.shape}")
|
| 44 |
+
H, W = rgba.shape[:2]
|
| 45 |
+
|
| 46 |
+
mask_bool = mask.astype(bool)
|
| 47 |
+
edge = mask_bool & ~binary_erosion(mask_bool, iterations=1, border_value=0)
|
| 48 |
+
edge_layer_arr = np.zeros((H, W, 4), dtype=np.uint8)
|
| 49 |
+
edge_layer_arr[edge] = MASK_EDGE_COLOR
|
| 50 |
+
|
| 51 |
+
canvas = Image.alpha_composite(
|
| 52 |
+
Image.fromarray(rgba, mode="RGBA"),
|
| 53 |
+
Image.fromarray(edge_layer_arr, mode="RGBA"),
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
draw = ImageDraw.Draw(canvas)
|
| 57 |
+
|
| 58 |
+
for parent_idx, child_idx in BONES:
|
| 59 |
+
py, px = skeleton.positions[parent_idx]
|
| 60 |
+
cy, cx = skeleton.positions[child_idx]
|
| 61 |
+
draw.line(
|
| 62 |
+
[(float(px), float(py)), (float(cx), float(cy))],
|
| 63 |
+
fill=BONE_COLOR,
|
| 64 |
+
width=BONE_WIDTH,
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
for i in range(skeleton.positions.shape[0]):
|
| 68 |
+
y, x = skeleton.positions[i]
|
| 69 |
+
r = JOINT_RADIUS
|
| 70 |
+
draw.ellipse(
|
| 71 |
+
[(float(x) - r, float(y) - r), (float(x) + r, float(y) + r)],
|
| 72 |
+
fill=JOINT_COLOR,
|
| 73 |
+
outline=JOINT_OUTLINE,
|
| 74 |
+
width=1,
|
| 75 |
+
)
|
| 76 |
+
if show_labels:
|
| 77 |
+
draw.text(
|
| 78 |
+
(float(x) + r + 2, float(y) - r),
|
| 79 |
+
JOINT_NAMES[i],
|
| 80 |
+
fill=(255, 255, 255, 255),
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
return canvas
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
__all__ = ["render_overlay"]
|
studio/rigging/parts.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.parts — piece-mask sidecar (Phase 3 / Session 3.5).
|
| 2 |
+
|
| 3 |
+
Each sprite gets two artifacts alongside its joints sidecar:
|
| 4 |
+
|
| 5 |
+
<sprite>.png (the sprite image)
|
| 6 |
+
<sprite>.joints.json (joint annotations, Session 2)
|
| 7 |
+
<sprite>.parts.png (uint8 grayscale; pixel = piece_id; 0 = background)
|
| 8 |
+
<sprite>.parts.json (piece index, bone binding, z-order, kind)
|
| 9 |
+
|
| 10 |
+
The parts.png is mode='L' so every editor opens and round-trips it cleanly. A
|
| 11 |
+
separate <sprite>.parts.preview.png with a color palette is generated for human
|
| 12 |
+
inspection; it is NOT load-bearing.
|
| 13 |
+
|
| 14 |
+
JSON format (versioned, forwards-compatible):
|
| 15 |
+
|
| 16 |
+
{
|
| 17 |
+
"schema_version": 1,
|
| 18 |
+
"image": "goblin.png",
|
| 19 |
+
"image_shape": [256, 256],
|
| 20 |
+
"pieces": {
|
| 21 |
+
"1": {"name": "torso", "bone": "neck", "z": 1, "kind": "body"},
|
| 22 |
+
...
|
| 23 |
+
"14": {"name": "sword", "bone": "r_wrist", "z": 9, "kind": "attachment"}
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
Piece IDs are 1-indexed. ID 0 is reserved for background (no piece).
|
| 28 |
+
|
| 29 |
+
`bone` is the joint name the piece pivots around. Under forward kinematics, the
|
| 30 |
+
piece rotates rigidly around that joint's position by that joint's world
|
| 31 |
+
rotation. `z` is the render order — higher draws on top. `kind` is "body" or
|
| 32 |
+
"attachment"; the default rigid-piece engine renders body-only.
|
| 33 |
+
"""
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import colorsys
|
| 37 |
+
import json
|
| 38 |
+
from dataclasses import dataclass
|
| 39 |
+
from pathlib import Path
|
| 40 |
+
from typing import Dict, Optional, Tuple
|
| 41 |
+
|
| 42 |
+
import numpy as np
|
| 43 |
+
from PIL import Image
|
| 44 |
+
|
| 45 |
+
from pixel_cursor.rigging import JOINT_INDEX, JOINT_NAMES
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
SCHEMA_VERSION: int = 1
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
CANONICAL_HUMANOID_PIECES: Tuple[Tuple[str, str, int, str], ...] = (
|
| 52 |
+
# (name, bone (joint name), z-order, kind)
|
| 53 |
+
("torso", "neck", 1, "body"),
|
| 54 |
+
("l_thigh", "l_hip", 2, "body"),
|
| 55 |
+
("r_thigh", "r_hip", 2, "body"),
|
| 56 |
+
("l_shin", "l_knee", 3, "body"),
|
| 57 |
+
("r_shin", "r_knee", 3, "body"),
|
| 58 |
+
("l_foot", "l_ankle", 4, "body"),
|
| 59 |
+
("r_foot", "r_ankle", 4, "body"),
|
| 60 |
+
("l_upper_arm", "l_shoulder", 5, "body"),
|
| 61 |
+
("r_upper_arm", "r_shoulder", 5, "body"),
|
| 62 |
+
("l_forearm", "l_elbow", 6, "body"),
|
| 63 |
+
("r_forearm", "r_elbow", 6, "body"),
|
| 64 |
+
("l_hand", "l_wrist", 7, "body"),
|
| 65 |
+
("r_hand", "r_wrist", 7, "body"),
|
| 66 |
+
("head", "head", 8, "body"),
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@dataclass(frozen=True)
|
| 71 |
+
class Piece:
|
| 72 |
+
piece_id: int
|
| 73 |
+
name: str
|
| 74 |
+
bone: str
|
| 75 |
+
z: int
|
| 76 |
+
kind: str
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@dataclass(frozen=True)
|
| 80 |
+
class PartsMask:
|
| 81 |
+
id_map: np.ndarray
|
| 82 |
+
pieces: Tuple[Piece, ...]
|
| 83 |
+
image_shape: Tuple[int, int]
|
| 84 |
+
|
| 85 |
+
def piece_by_name(self, name: str) -> Optional[Piece]:
|
| 86 |
+
for p in self.pieces:
|
| 87 |
+
if p.name == name:
|
| 88 |
+
return p
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
def piece_by_id(self, piece_id: int) -> Optional[Piece]:
|
| 92 |
+
for p in self.pieces:
|
| 93 |
+
if p.piece_id == piece_id:
|
| 94 |
+
return p
|
| 95 |
+
return None
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def body_pieces(self) -> Tuple[Piece, ...]:
|
| 99 |
+
return tuple(p for p in self.pieces if p.kind == "body")
|
| 100 |
+
|
| 101 |
+
@property
|
| 102 |
+
def attachment_pieces(self) -> Tuple[Piece, ...]:
|
| 103 |
+
return tuple(p for p in self.pieces if p.kind == "attachment")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def parts_png_path_for(image_path: Path | str) -> Path:
|
| 107 |
+
p = Path(image_path)
|
| 108 |
+
return p.with_name(p.stem + ".parts.png")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def parts_json_path_for(image_path: Path | str) -> Path:
|
| 112 |
+
p = Path(image_path)
|
| 113 |
+
return p.with_name(p.stem + ".parts.json")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def parts_preview_path_for(image_path: Path | str) -> Path:
|
| 117 |
+
p = Path(image_path)
|
| 118 |
+
return p.with_name(p.stem + ".parts.preview.png")
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def load_parts(image_path: Path | str) -> Optional[PartsMask]:
|
| 122 |
+
"""Return the PartsMask for image_path, or None if either sidecar is missing.
|
| 123 |
+
|
| 124 |
+
Raises ValueError if the JSON exists but is malformed or has a higher
|
| 125 |
+
schema_version than we support.
|
| 126 |
+
"""
|
| 127 |
+
png_path = parts_png_path_for(image_path)
|
| 128 |
+
json_path = parts_json_path_for(image_path)
|
| 129 |
+
if not png_path.exists() or not json_path.exists():
|
| 130 |
+
return None
|
| 131 |
+
|
| 132 |
+
data = json.loads(json_path.read_text())
|
| 133 |
+
version = int(data.get("schema_version", 1))
|
| 134 |
+
if version > SCHEMA_VERSION:
|
| 135 |
+
raise ValueError(
|
| 136 |
+
f"parts sidecar schema_version {version} > supported {SCHEMA_VERSION} "
|
| 137 |
+
f"({json_path})"
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
shape_field = data["image_shape"]
|
| 141 |
+
H, W = int(shape_field[0]), int(shape_field[1])
|
| 142 |
+
|
| 143 |
+
img = Image.open(png_path)
|
| 144 |
+
if img.mode != "L":
|
| 145 |
+
img = img.convert("L")
|
| 146 |
+
arr = np.asarray(img, dtype=np.uint8)
|
| 147 |
+
if arr.shape != (H, W):
|
| 148 |
+
raise ValueError(
|
| 149 |
+
f"parts.png shape {arr.shape} != image_shape {(H, W)} ({png_path})"
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
pieces_field = data.get("pieces", {})
|
| 153 |
+
pieces = []
|
| 154 |
+
for pid_str, entry in pieces_field.items():
|
| 155 |
+
pid = int(pid_str)
|
| 156 |
+
if pid == 0:
|
| 157 |
+
raise ValueError(
|
| 158 |
+
f"piece_id 0 is reserved for background ({json_path})"
|
| 159 |
+
)
|
| 160 |
+
if pid < 0 or pid > 255:
|
| 161 |
+
raise ValueError(
|
| 162 |
+
f"piece_id {pid} out of uint8 range ({json_path})"
|
| 163 |
+
)
|
| 164 |
+
bone = entry["bone"]
|
| 165 |
+
if bone not in JOINT_INDEX:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
f"piece '{entry.get('name', pid)}' bone '{bone}' is not a known "
|
| 168 |
+
f"joint ({json_path}); known: {list(JOINT_NAMES)}"
|
| 169 |
+
)
|
| 170 |
+
pieces.append(Piece(
|
| 171 |
+
piece_id=pid,
|
| 172 |
+
name=str(entry["name"]),
|
| 173 |
+
bone=bone,
|
| 174 |
+
z=int(entry.get("z", 0)),
|
| 175 |
+
kind=str(entry.get("kind", "body")),
|
| 176 |
+
))
|
| 177 |
+
|
| 178 |
+
return PartsMask(
|
| 179 |
+
id_map=arr,
|
| 180 |
+
pieces=tuple(pieces),
|
| 181 |
+
image_shape=(H, W),
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def save_parts(
|
| 186 |
+
image_path: Path | str,
|
| 187 |
+
id_map: np.ndarray,
|
| 188 |
+
pieces: Tuple[Piece, ...],
|
| 189 |
+
*,
|
| 190 |
+
image_name: Optional[str] = None,
|
| 191 |
+
) -> Tuple[Path, Path]:
|
| 192 |
+
"""Write <sprite>.parts.png + <sprite>.parts.json. Returns (png_path, json_path)."""
|
| 193 |
+
if id_map.ndim != 2:
|
| 194 |
+
raise ValueError(f"id_map must be 2D, got {id_map.shape}")
|
| 195 |
+
if id_map.dtype != np.uint8:
|
| 196 |
+
raise ValueError(f"id_map must be uint8, got {id_map.dtype}")
|
| 197 |
+
png_path = parts_png_path_for(image_path)
|
| 198 |
+
json_path = parts_json_path_for(image_path)
|
| 199 |
+
name = image_name if image_name is not None else Path(image_path).name
|
| 200 |
+
|
| 201 |
+
Image.fromarray(id_map, mode="L").save(png_path)
|
| 202 |
+
|
| 203 |
+
payload = {
|
| 204 |
+
"schema_version": SCHEMA_VERSION,
|
| 205 |
+
"image": name,
|
| 206 |
+
"image_shape": [int(id_map.shape[0]), int(id_map.shape[1])],
|
| 207 |
+
"pieces": {
|
| 208 |
+
str(p.piece_id): {
|
| 209 |
+
"name": p.name,
|
| 210 |
+
"bone": p.bone,
|
| 211 |
+
"z": p.z,
|
| 212 |
+
"kind": p.kind,
|
| 213 |
+
}
|
| 214 |
+
for p in pieces
|
| 215 |
+
},
|
| 216 |
+
}
|
| 217 |
+
json_path.write_text(json.dumps(payload, indent=2, sort_keys=False) + "\n")
|
| 218 |
+
return png_path, json_path
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def save_preview(
|
| 222 |
+
image_path: Path | str,
|
| 223 |
+
parts: PartsMask,
|
| 224 |
+
) -> Path:
|
| 225 |
+
"""Write a color-coded RGBA preview alongside the parts sidecar.
|
| 226 |
+
|
| 227 |
+
Pure operator convenience — not load-bearing. Background is transparent.
|
| 228 |
+
"""
|
| 229 |
+
H, W = parts.image_shape
|
| 230 |
+
rgba = np.zeros((H, W, 4), dtype=np.uint8)
|
| 231 |
+
palette = _build_preview_palette(parts.pieces)
|
| 232 |
+
for piece in parts.pieces:
|
| 233 |
+
mask = parts.id_map == piece.piece_id
|
| 234 |
+
if not mask.any():
|
| 235 |
+
continue
|
| 236 |
+
rgba[mask] = palette[piece.piece_id]
|
| 237 |
+
out_path = parts_preview_path_for(image_path)
|
| 238 |
+
Image.fromarray(rgba, mode="RGBA").save(out_path)
|
| 239 |
+
return out_path
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def _build_preview_palette(
|
| 243 |
+
pieces: Tuple[Piece, ...],
|
| 244 |
+
) -> Dict[int, Tuple[int, int, int, int]]:
|
| 245 |
+
palette: Dict[int, Tuple[int, int, int, int]] = {}
|
| 246 |
+
n = max(1, len(pieces))
|
| 247 |
+
for i, p in enumerate(pieces):
|
| 248 |
+
hue = (i / n) % 1.0
|
| 249 |
+
sat = 0.65
|
| 250 |
+
val = 0.95 if p.kind == "body" else 0.55
|
| 251 |
+
r, g, b = colorsys.hsv_to_rgb(hue, sat, val)
|
| 252 |
+
palette[p.piece_id] = (int(r * 255), int(g * 255), int(b * 255), 220)
|
| 253 |
+
return palette
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def canonical_humanoid_pieces(
|
| 257 |
+
extras: Tuple[Tuple[str, str, int, str], ...] = (),
|
| 258 |
+
) -> Tuple[Piece, ...]:
|
| 259 |
+
"""Return the canonical 13 humanoid pieces plus any extras (attachments).
|
| 260 |
+
|
| 261 |
+
extras: iterable of (name, bone, z, kind) tuples to append. Piece IDs are
|
| 262 |
+
assigned sequentially starting from 1.
|
| 263 |
+
"""
|
| 264 |
+
pieces = []
|
| 265 |
+
next_id = 1
|
| 266 |
+
for name, bone, z, kind in CANONICAL_HUMANOID_PIECES:
|
| 267 |
+
pieces.append(Piece(piece_id=next_id, name=name, bone=bone, z=z, kind=kind))
|
| 268 |
+
next_id += 1
|
| 269 |
+
for name, bone, z, kind in extras:
|
| 270 |
+
pieces.append(Piece(piece_id=next_id, name=name, bone=bone, z=z, kind=kind))
|
| 271 |
+
next_id += 1
|
| 272 |
+
return tuple(pieces)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
__all__ = [
|
| 276 |
+
"SCHEMA_VERSION",
|
| 277 |
+
"CANONICAL_HUMANOID_PIECES",
|
| 278 |
+
"Piece",
|
| 279 |
+
"PartsMask",
|
| 280 |
+
"parts_png_path_for",
|
| 281 |
+
"parts_json_path_for",
|
| 282 |
+
"parts_preview_path_for",
|
| 283 |
+
"load_parts",
|
| 284 |
+
"save_parts",
|
| 285 |
+
"save_preview",
|
| 286 |
+
"canonical_humanoid_pieces",
|
| 287 |
+
]
|
studio/rigging/rigid_piece.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.rigid_piece — rigid-piece backward-warp engine (Session 3.5).
|
| 2 |
+
|
| 3 |
+
Each piece rotates rigidly around its bone joint's position by that joint's
|
| 4 |
+
world rotation (computed via the same FK pass the LBS engine uses). Pixels are
|
| 5 |
+
sampled by backward warp: for each output pixel, the inverse transform maps to
|
| 6 |
+
a source pixel; if the source pixel belongs to this piece in the rest parts.png,
|
| 7 |
+
its color is copied.
|
| 8 |
+
|
| 9 |
+
Why backward-warp instead of forward-splat:
|
| 10 |
+
- No gaps when a piece under-samples its output region after rotation.
|
| 11 |
+
- z-order is the natural occlusion model — paint back-to-front, higher z wins.
|
| 12 |
+
- Nearest-neighbor membership check matches sprite-art pixel discipline.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from typing import Iterable, Mapping, Tuple
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
|
| 20 |
+
from pixel_cursor.rigging import JOINT_INDEX, Skeleton
|
| 21 |
+
|
| 22 |
+
from .deform import forward_kinematics
|
| 23 |
+
from .parts import PartsMask
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def deform_sprite_rigid(
|
| 27 |
+
image: np.ndarray,
|
| 28 |
+
rest: Skeleton,
|
| 29 |
+
parts: PartsMask,
|
| 30 |
+
*,
|
| 31 |
+
local_rotations: Mapping[str, float] | None = None,
|
| 32 |
+
include_kinds: Tuple[str, ...] = ("body",),
|
| 33 |
+
exclude_pieces: Iterable[str] = (),
|
| 34 |
+
) -> Tuple[np.ndarray, np.ndarray]:
|
| 35 |
+
"""Render a posed configuration via rigid-piece backward warp.
|
| 36 |
+
|
| 37 |
+
Args:
|
| 38 |
+
image: source sprite, shape (H, W, 3 or 4) uint8.
|
| 39 |
+
rest: rest-pose skeleton (image_shape must equal image's H, W).
|
| 40 |
+
parts: piece-mask sidecar (id_map shape must equal image's H, W).
|
| 41 |
+
local_rotations: optional dict mapping joint_name -> local rotation (rad).
|
| 42 |
+
include_kinds: piece.kind values to render. Default ('body',) — attachments
|
| 43 |
+
are handled by the composite layer (Session 4).
|
| 44 |
+
exclude_pieces: piece names to skip (e.g., ('sword',) for swordless render).
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
(rgba (H, W, 4) uint8, mask (H, W) bool)
|
| 48 |
+
|
| 49 |
+
The output is composed back-to-front by z; higher z overwrites lower. Every
|
| 50 |
+
output pixel is a pixel-perfect copy of exactly one source pixel (or fully
|
| 51 |
+
transparent) — this is the 'rigid' invariant.
|
| 52 |
+
"""
|
| 53 |
+
H, W = parts.image_shape
|
| 54 |
+
if image.shape[:2] != (H, W):
|
| 55 |
+
raise ValueError(
|
| 56 |
+
f"image shape {image.shape[:2]} != parts.image_shape {(H, W)}"
|
| 57 |
+
)
|
| 58 |
+
if rest.image_shape != (H, W):
|
| 59 |
+
raise ValueError(
|
| 60 |
+
f"rest.image_shape {rest.image_shape} != parts.image_shape {(H, W)}"
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
if image.ndim == 3 and image.shape[-1] == 4:
|
| 64 |
+
rgba = image
|
| 65 |
+
elif image.ndim == 3 and image.shape[-1] == 3:
|
| 66 |
+
rgba = np.concatenate(
|
| 67 |
+
[image, np.full((H, W, 1), 255, dtype=np.uint8)], axis=-1
|
| 68 |
+
)
|
| 69 |
+
elif image.ndim == 2:
|
| 70 |
+
gray = image
|
| 71 |
+
rgba = np.stack(
|
| 72 |
+
[gray, gray, gray, np.full_like(gray, 255)], axis=-1
|
| 73 |
+
)
|
| 74 |
+
else:
|
| 75 |
+
raise ValueError(f"unsupported image shape {image.shape}")
|
| 76 |
+
|
| 77 |
+
deformed_pos, world_rot = forward_kinematics(rest, local_rotations or {})
|
| 78 |
+
|
| 79 |
+
out_rgba = np.zeros((H, W, 4), dtype=np.uint8)
|
| 80 |
+
out_mask = np.zeros((H, W), dtype=bool)
|
| 81 |
+
|
| 82 |
+
yy, xx = np.mgrid[0:H, 0:W].astype(np.float32)
|
| 83 |
+
|
| 84 |
+
exclude = set(exclude_pieces)
|
| 85 |
+
selected = [
|
| 86 |
+
p for p in parts.pieces
|
| 87 |
+
if p.kind in include_kinds and p.name not in exclude
|
| 88 |
+
]
|
| 89 |
+
selected.sort(key=lambda p: p.z)
|
| 90 |
+
|
| 91 |
+
for piece in selected:
|
| 92 |
+
bone_idx = JOINT_INDEX[piece.bone]
|
| 93 |
+
pivot_rest_y = float(rest.positions[bone_idx, 0])
|
| 94 |
+
pivot_rest_x = float(rest.positions[bone_idx, 1])
|
| 95 |
+
pivot_posed_y = float(deformed_pos[bone_idx, 0])
|
| 96 |
+
pivot_posed_x = float(deformed_pos[bone_idx, 1])
|
| 97 |
+
theta = float(world_rot[bone_idx])
|
| 98 |
+
|
| 99 |
+
c = np.cos(-theta)
|
| 100 |
+
s = np.sin(-theta)
|
| 101 |
+
|
| 102 |
+
dy = yy - pivot_posed_y
|
| 103 |
+
dx = xx - pivot_posed_x
|
| 104 |
+
src_y = pivot_rest_y + c * dy - s * dx
|
| 105 |
+
src_x = pivot_rest_x + s * dy + c * dx
|
| 106 |
+
|
| 107 |
+
sy_i = np.round(src_y).astype(np.int32)
|
| 108 |
+
sx_i = np.round(src_x).astype(np.int32)
|
| 109 |
+
in_bounds = (sy_i >= 0) & (sy_i < H) & (sx_i >= 0) & (sx_i < W)
|
| 110 |
+
sy_safe = np.clip(sy_i, 0, H - 1)
|
| 111 |
+
sx_safe = np.clip(sx_i, 0, W - 1)
|
| 112 |
+
|
| 113 |
+
membership = parts.id_map[sy_safe, sx_safe] == piece.piece_id
|
| 114 |
+
src_alpha = rgba[sy_safe, sx_safe, 3] > 0
|
| 115 |
+
valid = in_bounds & membership & src_alpha
|
| 116 |
+
if not valid.any():
|
| 117 |
+
continue
|
| 118 |
+
out_rgba[valid] = rgba[sy_safe[valid], sx_safe[valid]]
|
| 119 |
+
out_mask |= valid
|
| 120 |
+
|
| 121 |
+
return out_rgba, out_mask
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
__all__ = ["deform_sprite_rigid"]
|
studio/rigging/segment.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.segment — flood-fill-from-corners cartoon segmentation.
|
| 2 |
+
|
| 3 |
+
Real alpha channel short-circuits the heuristic: `alpha > 127` is the mask.
|
| 4 |
+
Otherwise: connected components of the "background-similar" mask, take the
|
| 5 |
+
union of components that touch any of the 4 image corners, invert.
|
| 6 |
+
|
| 7 |
+
Deterministic, numpy + scipy.ndimage.label, no model imports.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
from scipy.ndimage import label
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
DEFAULT_TOLERANCE: int = 15
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def flood_fill_segment(
|
| 19 |
+
image: np.ndarray,
|
| 20 |
+
*,
|
| 21 |
+
tolerance: int = DEFAULT_TOLERANCE,
|
| 22 |
+
alpha_threshold: int = 127,
|
| 23 |
+
) -> np.ndarray:
|
| 24 |
+
"""Return a boolean (H, W) foreground mask.
|
| 25 |
+
|
| 26 |
+
image: (H, W) uint8 grayscale, (H, W, 3) RGB uint8, or (H, W, 4) RGBA uint8.
|
| 27 |
+
tolerance: per-channel Chebyshev tolerance for "background-similar" pixels.
|
| 28 |
+
alpha_threshold: cutoff for the RGBA fast path (only used if the alpha
|
| 29 |
+
channel actually varies across the image).
|
| 30 |
+
"""
|
| 31 |
+
if image.ndim == 3 and image.shape[-1] == 4:
|
| 32 |
+
alpha = image[..., 3]
|
| 33 |
+
if alpha.min() < 255 and alpha.max() > 0 and alpha.max() != alpha.min():
|
| 34 |
+
return alpha > alpha_threshold
|
| 35 |
+
image = image[..., :3]
|
| 36 |
+
|
| 37 |
+
if image.ndim == 2:
|
| 38 |
+
rgb = np.stack([image, image, image], axis=-1)
|
| 39 |
+
elif image.ndim == 3 and image.shape[-1] == 3:
|
| 40 |
+
rgb = image
|
| 41 |
+
else:
|
| 42 |
+
raise ValueError(f"unsupported image shape {image.shape}")
|
| 43 |
+
|
| 44 |
+
H, W, _ = rgb.shape
|
| 45 |
+
rgb_i16 = rgb.astype(np.int16)
|
| 46 |
+
|
| 47 |
+
corners = np.stack(
|
| 48 |
+
[rgb_i16[0, 0], rgb_i16[0, -1], rgb_i16[-1, 0], rgb_i16[-1, -1]],
|
| 49 |
+
axis=0,
|
| 50 |
+
)
|
| 51 |
+
bg_color = corners.mean(axis=0)
|
| 52 |
+
diff = np.abs(rgb_i16 - bg_color).max(axis=-1)
|
| 53 |
+
similar = diff <= tolerance
|
| 54 |
+
|
| 55 |
+
labels, _ = label(similar.astype(np.uint8), structure=np.ones((3, 3), dtype=np.int8))
|
| 56 |
+
|
| 57 |
+
bg_label_ids = set()
|
| 58 |
+
for y, x in ((0, 0), (0, W - 1), (H - 1, 0), (H - 1, W - 1)):
|
| 59 |
+
L = int(labels[y, x])
|
| 60 |
+
if L > 0:
|
| 61 |
+
bg_label_ids.add(L)
|
| 62 |
+
|
| 63 |
+
if not bg_label_ids:
|
| 64 |
+
return np.ones((H, W), dtype=bool)
|
| 65 |
+
|
| 66 |
+
bg_mask = np.isin(labels, np.asarray(sorted(bg_label_ids), dtype=labels.dtype))
|
| 67 |
+
return ~bg_mask
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
__all__ = ["flood_fill_segment", "DEFAULT_TOLERANCE"]
|
studio/rigging/sidecar.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.sidecar — operator-annotation JSON loader + saver.
|
| 2 |
+
|
| 3 |
+
Sidecar format (versioned, forwards-compatible):
|
| 4 |
+
|
| 5 |
+
{
|
| 6 |
+
"schema_version": 1,
|
| 7 |
+
"image": "goblin.png",
|
| 8 |
+
"image_shape": [256, 256],
|
| 9 |
+
"joints": {
|
| 10 |
+
"head": {"y": 50, "x": 128, "confidence": 1.0},
|
| 11 |
+
"neck": {"y": 110, "x": 128},
|
| 12 |
+
...
|
| 13 |
+
}
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
Sidecar path convention: `<sprite>.joints.json` next to `<sprite>.png`.
|
| 17 |
+
|
| 18 |
+
Loader is lenient: missing joints default to image-center with confidence 0.0,
|
| 19 |
+
so the overlay renders them clearly out-of-place — the operator drags them to
|
| 20 |
+
the correct positions by editing the JSON.
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import json
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
from typing import Optional, Tuple
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
|
| 30 |
+
from pixel_cursor.rigging import JOINT_INDEX, JOINT_NAMES, NUM_JOINTS, Skeleton
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
SCHEMA_VERSION: int = 1
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def sidecar_path_for(image_path: Path | str) -> Path:
|
| 37 |
+
p = Path(image_path)
|
| 38 |
+
return p.with_suffix(".joints.json")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def load_sidecar(
|
| 42 |
+
image_path: Path | str,
|
| 43 |
+
*,
|
| 44 |
+
image_shape: Optional[Tuple[int, int]] = None,
|
| 45 |
+
) -> Optional[Skeleton]:
|
| 46 |
+
"""Return a Skeleton from the sidecar next to image_path, or None if absent.
|
| 47 |
+
|
| 48 |
+
Missing or malformed joint entries default to image center with confidence 0.
|
| 49 |
+
"""
|
| 50 |
+
sidecar = sidecar_path_for(image_path)
|
| 51 |
+
if not sidecar.exists():
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
data = json.loads(sidecar.read_text())
|
| 55 |
+
version = int(data.get("schema_version", 1))
|
| 56 |
+
if version > SCHEMA_VERSION:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
f"sidecar schema_version {version} > supported {SCHEMA_VERSION} "
|
| 59 |
+
f"({sidecar})"
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
shape_field = data.get("image_shape")
|
| 63 |
+
if shape_field is not None:
|
| 64 |
+
H, W = int(shape_field[0]), int(shape_field[1])
|
| 65 |
+
elif image_shape is not None:
|
| 66 |
+
H, W = image_shape
|
| 67 |
+
else:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
f"sidecar {sidecar} has no image_shape; pass image_shape= to load_sidecar"
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
joints_field = data.get("joints", {})
|
| 73 |
+
positions = np.zeros((NUM_JOINTS, 2), dtype=np.float32)
|
| 74 |
+
confidence = np.zeros(NUM_JOINTS, dtype=np.float32)
|
| 75 |
+
|
| 76 |
+
for name in JOINT_NAMES:
|
| 77 |
+
entry = joints_field.get(name)
|
| 78 |
+
idx = JOINT_INDEX[name]
|
| 79 |
+
if entry is None:
|
| 80 |
+
positions[idx] = (H / 2.0, W / 2.0)
|
| 81 |
+
confidence[idx] = 0.0
|
| 82 |
+
continue
|
| 83 |
+
positions[idx, 0] = float(entry["y"])
|
| 84 |
+
positions[idx, 1] = float(entry["x"])
|
| 85 |
+
confidence[idx] = float(entry.get("confidence", 1.0))
|
| 86 |
+
|
| 87 |
+
return Skeleton(positions=positions, confidence=confidence, image_shape=(H, W))
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def save_sidecar(
|
| 91 |
+
image_path: Path | str,
|
| 92 |
+
skeleton: Skeleton,
|
| 93 |
+
*,
|
| 94 |
+
image_name: Optional[str] = None,
|
| 95 |
+
) -> Path:
|
| 96 |
+
sidecar = sidecar_path_for(image_path)
|
| 97 |
+
name = image_name if image_name is not None else Path(image_path).name
|
| 98 |
+
payload = {
|
| 99 |
+
"schema_version": SCHEMA_VERSION,
|
| 100 |
+
"image": name,
|
| 101 |
+
"image_shape": list(skeleton.image_shape),
|
| 102 |
+
"joints": {
|
| 103 |
+
joint_name: {
|
| 104 |
+
"y": round(float(skeleton.positions[i, 0]), 2),
|
| 105 |
+
"x": round(float(skeleton.positions[i, 1]), 2),
|
| 106 |
+
"confidence": round(float(skeleton.confidence[i]), 3),
|
| 107 |
+
}
|
| 108 |
+
for i, joint_name in enumerate(JOINT_NAMES)
|
| 109 |
+
},
|
| 110 |
+
}
|
| 111 |
+
sidecar.write_text(json.dumps(payload, indent=2, sort_keys=False) + "\n")
|
| 112 |
+
return sidecar
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def sidecar_template(
|
| 116 |
+
image_shape: Tuple[int, int],
|
| 117 |
+
image_name: str = "",
|
| 118 |
+
) -> dict:
|
| 119 |
+
"""Return a template dict the operator can write to disk and fill in."""
|
| 120 |
+
H, W = image_shape
|
| 121 |
+
cy, cx = H / 2.0, W / 2.0
|
| 122 |
+
return {
|
| 123 |
+
"schema_version": SCHEMA_VERSION,
|
| 124 |
+
"image": image_name,
|
| 125 |
+
"image_shape": [H, W],
|
| 126 |
+
"joints": {
|
| 127 |
+
name: {"y": cy, "x": cx, "confidence": 0.0}
|
| 128 |
+
for name in JOINT_NAMES
|
| 129 |
+
},
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
__all__ = [
|
| 134 |
+
"SCHEMA_VERSION",
|
| 135 |
+
"sidecar_path_for",
|
| 136 |
+
"load_sidecar",
|
| 137 |
+
"save_sidecar",
|
| 138 |
+
"sidecar_template",
|
| 139 |
+
]
|
studio/rigging/skin.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""studio.rigging.skin — inverse-distance-falloff mesh skinning weights.
|
| 2 |
+
|
| 3 |
+
For each foreground pixel and each bone, weight ∝ 1 / distance_to_bone^p,
|
| 4 |
+
normalised across bones to sum to 1 per pixel. Background pixels get zero
|
| 5 |
+
weight on every bone.
|
| 6 |
+
|
| 7 |
+
This is the "fast / good-enough" baseline picked in PHASE_3_RESEARCH.md.
|
| 8 |
+
Heat-diffusion skinning (better crease quality) is the upgrade path for
|
| 9 |
+
Session 4 if creases hurt visual quality.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
|
| 15 |
+
from pixel_cursor.rigging import BONES, NUM_JOINTS, Skeleton
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
DEFAULT_FALLOFF: float = 4.0
|
| 19 |
+
DEFAULT_EPSILON: float = 0.5
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _point_segment_distance(
|
| 23 |
+
points: np.ndarray, a: np.ndarray, b: np.ndarray
|
| 24 |
+
) -> np.ndarray:
|
| 25 |
+
"""Distance from each point to the closed segment AB.
|
| 26 |
+
|
| 27 |
+
points: shape (..., 2) in (y, x)
|
| 28 |
+
a, b: shape (2,) endpoints
|
| 29 |
+
Returns shape (...)
|
| 30 |
+
"""
|
| 31 |
+
ab = b - a
|
| 32 |
+
ap = points - a
|
| 33 |
+
ab_len_sq = float((ab * ab).sum())
|
| 34 |
+
if ab_len_sq < 1e-6:
|
| 35 |
+
return np.linalg.norm(ap, axis=-1)
|
| 36 |
+
t = (ap * ab).sum(axis=-1) / ab_len_sq
|
| 37 |
+
t = np.clip(t, 0.0, 1.0)
|
| 38 |
+
closest = a + t[..., None] * ab
|
| 39 |
+
return np.linalg.norm(points - closest, axis=-1)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def compute_skin_weights(
|
| 43 |
+
skeleton: Skeleton,
|
| 44 |
+
mask: np.ndarray,
|
| 45 |
+
*,
|
| 46 |
+
falloff: float = DEFAULT_FALLOFF,
|
| 47 |
+
epsilon: float = DEFAULT_EPSILON,
|
| 48 |
+
) -> np.ndarray:
|
| 49 |
+
"""Return (H, W, N_BONES) inverse-distance-falloff skinning weights.
|
| 50 |
+
|
| 51 |
+
Properties:
|
| 52 |
+
- Each foreground pixel's weights sum to 1.0 (partition of unity).
|
| 53 |
+
- Each background pixel's weights are exactly 0 across all bones.
|
| 54 |
+
- Deterministic: same skeleton + mask → same weights, bitwise.
|
| 55 |
+
"""
|
| 56 |
+
if mask.ndim != 2:
|
| 57 |
+
raise ValueError(f"mask must be 2D, got {mask.shape}")
|
| 58 |
+
if skeleton.image_shape != mask.shape:
|
| 59 |
+
raise ValueError(
|
| 60 |
+
f"skeleton.image_shape={skeleton.image_shape} != mask.shape={mask.shape}"
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
H, W = mask.shape
|
| 64 |
+
ys, xs = np.indices((H, W))
|
| 65 |
+
coords = np.stack([ys, xs], axis=-1).astype(np.float32)
|
| 66 |
+
|
| 67 |
+
n_bones = len(BONES)
|
| 68 |
+
dists = np.empty((H, W, n_bones), dtype=np.float32)
|
| 69 |
+
rest = skeleton.positions
|
| 70 |
+
for b, (parent_idx, child_idx) in enumerate(BONES):
|
| 71 |
+
dists[..., b] = _point_segment_distance(
|
| 72 |
+
coords, rest[parent_idx], rest[child_idx]
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
inv = 1.0 / np.maximum(dists, epsilon) ** falloff
|
| 76 |
+
total = inv.sum(axis=-1, keepdims=True)
|
| 77 |
+
weights = inv / np.maximum(total, 1e-12)
|
| 78 |
+
|
| 79 |
+
mask_bool = mask.astype(bool)[..., None]
|
| 80 |
+
weights = np.where(mask_bool, weights, 0.0).astype(np.float32)
|
| 81 |
+
return weights
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
__all__ = ["compute_skin_weights", "DEFAULT_FALLOFF", "DEFAULT_EPSILON"]
|
studio/spritesheet.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FrameStack -> sprite sheet PNG.
|
| 2 |
+
|
| 3 |
+
A sprite sheet is a single PNG containing N animation frames in a grid. This is
|
| 4 |
+
the idiomatic asset format for 2D game engines (Babylon.js `SpriteManager`,
|
| 5 |
+
Phaser, PixiJS) — no codec dependency, instant load, deterministic playback.
|
| 6 |
+
|
| 7 |
+
Convention: frames packed left-to-right, top-to-bottom. Default 4 columns gives
|
| 8 |
+
square-ish output for 16-frame stacks (4x4 = 16 cells).
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
from PIL import Image as PILImage
|
| 15 |
+
|
| 16 |
+
from pixel_cursor import FrameStack
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def frame_stack_to_sprite_sheet(
|
| 20 |
+
stack: FrameStack,
|
| 21 |
+
cols: int = 4,
|
| 22 |
+
bg_rgba: tuple[int, int, int, int] = (0, 0, 0, 0),
|
| 23 |
+
) -> PILImage.Image:
|
| 24 |
+
"""Pack T frames into a (rows*cols)-cell grid PNG. RGBA output."""
|
| 25 |
+
t, h, w, c = stack.frames.shape
|
| 26 |
+
if c not in (3, 4):
|
| 27 |
+
raise ValueError(f"expected 3 or 4 channels; got shape {stack.frames.shape}")
|
| 28 |
+
rows = (t + cols - 1) // cols
|
| 29 |
+
sheet_w = w * cols
|
| 30 |
+
sheet_h = h * rows
|
| 31 |
+
|
| 32 |
+
sheet = np.zeros((sheet_h, sheet_w, 4), dtype=np.uint8)
|
| 33 |
+
sheet[:, :] = bg_rgba
|
| 34 |
+
|
| 35 |
+
for idx in range(t):
|
| 36 |
+
r, k = divmod(idx, cols)
|
| 37 |
+
y0, x0 = r * h, k * w
|
| 38 |
+
frame = stack.frames[idx]
|
| 39 |
+
if c == 3:
|
| 40 |
+
sheet[y0 : y0 + h, x0 : x0 + w, :3] = frame
|
| 41 |
+
sheet[y0 : y0 + h, x0 : x0 + w, 3] = 255
|
| 42 |
+
else:
|
| 43 |
+
sheet[y0 : y0 + h, x0 : x0 + w, :] = frame
|
| 44 |
+
|
| 45 |
+
return PILImage.fromarray(sheet, mode="RGBA")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def write_sprite_sheet(
|
| 49 |
+
stack: FrameStack,
|
| 50 |
+
out_path: Path | str,
|
| 51 |
+
cols: int = 4,
|
| 52 |
+
bg_rgba: tuple[int, int, int, int] = (0, 0, 0, 0),
|
| 53 |
+
) -> Path:
|
| 54 |
+
out = Path(out_path)
|
| 55 |
+
img = frame_stack_to_sprite_sheet(stack, cols=cols, bg_rgba=bg_rgba)
|
| 56 |
+
img.save(out, optimize=True)
|
| 57 |
+
return out
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def sheet_metadata(stack: FrameStack, cols: int = 4) -> dict:
|
| 61 |
+
"""Return a manifest describing the sheet (cell size, count, fps).
|
| 62 |
+
|
| 63 |
+
Game engines need this to play back. Babylon SpriteManager expects
|
| 64 |
+
`cellWidth, cellHeight, capacity (total cells)`.
|
| 65 |
+
"""
|
| 66 |
+
t, h, w, _ = stack.frames.shape
|
| 67 |
+
rows = (t + cols - 1) // cols
|
| 68 |
+
return {
|
| 69 |
+
"frames": t,
|
| 70 |
+
"cols": cols,
|
| 71 |
+
"rows": rows,
|
| 72 |
+
"cell_width": w,
|
| 73 |
+
"cell_height": h,
|
| 74 |
+
"sheet_width": w * cols,
|
| 75 |
+
"sheet_height": h * rows,
|
| 76 |
+
"fps": stack.fps,
|
| 77 |
+
"loop_duration_ms": int(round(1000 * t / max(1, stack.fps))),
|
| 78 |
+
}
|