diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..f8ddca43c5cbc505d7bcc7d69eb59dd5f64d45db --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.npz filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c6d14693bb3a0d35ec55d39786acafd83aaea610 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir "gradio[mcp]" Pillow huggingface-hub safetensors einops numpy tqdm + +WORKDIR /app +COPY . . + +EXPOSE 7860 +CMD ["python", "app.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e6f64a6b7110769ed65712b175b8ac00f3a50be6 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +--- +title: FE2E Depth + Normal (CPU) +emoji: 🗺️ +colorFrom: blue +colorTo: green +sdk: docker +pinned: false +license: mit +short_description: Depth + Normal estimation from a single image on CPU +tags: + - depth-estimation + - normal-estimation + - cpu + - mcp-server +--- + +# FE2E: Depth + Normal Estimation (CPU) + +Estimate depth and surface normals from a single image using [FE2E](https://github.com/AMAP-ML/FE2E) (CVPR 2026). + +- Model: Step1X-Edit DiT + LDRN LoRA (FP8 -> dynamic INT8 on CPU) +- Single denoise step (not iterative diffusion) +- Outputs both depth AND normal maps simultaneously diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e6d94141cc6fb4f80de4b439a61112fad71e3219 --- /dev/null +++ b/app.py @@ -0,0 +1,190 @@ +"""FE2E: Depth + Normal estimation from a single image (CPU, FP32 with dynamic INT8)""" +from __future__ import annotations + +import gc +import os +import sys +import time + +import numpy as np +import torch + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +MODELS_DIR = "/tmp/fe2e_models" +os.makedirs(MODELS_DIR, exist_ok=True) + + +class Args: + prompt_type = "empty" + single_denoise = True + empty_prompt_cache = os.path.join(os.path.dirname(os.path.abspath(__file__)), "latent", "no_info.npz") + norm_type = "ln" + + +def _download_models(): + from huggingface_hub import hf_hub_download + import shutil + + token = os.environ.get("HF_TOKEN") + files = { + "dit": ("rkfg/Step1X-Edit-FP8", "step1x-edit-i1258-FP8.safetensors"), + "vae": ("exander/FE2E", "pretrain/vae.safetensors"), + "lora": ("exander/FE2E", "LDRN.safetensors"), + } + paths = {} + for key, (repo, filename) in files.items(): + basename = os.path.basename(filename) + dest = os.path.join(MODELS_DIR, basename) + if not os.path.exists(dest): + print(f"[init] Downloading {repo}/{filename}...") + src = hf_hub_download(repo, filename, token=token) + shutil.copy2(src, dest) + print(f"[init] {basename}: {os.path.getsize(dest)/1024/1024:.0f} MB") + paths[key] = dest + return paths + + +def _load_generator(paths): + """Load model: FP8 weights cast to FP32 for CPU, with LoRA merged.""" + from infer.inference import ImageGenerator + + args = Args() + print("[init] Loading model (FP8 -> FP32 on CPU)...") + t0 = time.time() + + generator = ImageGenerator( + dit_path=paths["dit"], + ae_path=paths["vae"], + quantized=True, + offload=False, + lora=paths["lora"], + device="cpu", + args=args, + ) + # FP8 tensors can't compute on CPU, cast to FP32 + generator.dit = generator.dit.float() + generator.ae = generator.ae.float() + + # Dynamic INT8 quantization for linear layers (biggest speedup on CPU) + generator.dit = torch.quantization.quantize_dynamic( + generator.dit, {torch.nn.Linear}, dtype=torch.qint8 + ) + + elapsed = time.time() - t0 + print(f"[init] Model loaded + quantized in {elapsed:.0f}s") + gc.collect() + return generator + + +GENERATOR = None + + +def generate(image): + """Estimate depth and surface normals from a single image. + + Args: + image: Input image (PIL Image or filepath). + + Returns: + tuple: (depth_map, normal_map, status_message) + """ + import gradio as gr + from PIL import Image + from torchvision.transforms import functional as F + + global GENERATOR + if GENERATOR is None: + paths = _download_models() + GENERATOR = _load_generator(paths) + + if image is None: + raise gr.Error("Please upload an image.") + + if isinstance(image, str): + image = Image.open(image).convert("RGB") + elif not isinstance(image, Image.Image): + image = Image.fromarray(image).convert("RGB") + + args = Args() + print(f"[gen] Input: {image.size}, starting inference...") + t0 = time.time() + + with torch.inference_mode(): + images, Lpred, Rpred = GENERATOR.generate_image( + prompt="", + negative_prompt="", + ref_images=image, + num_samples=1, + num_steps=1, + cfg_guidance=6.0, + seed=42, + show_progress=True, + args=args, + ) + + elapsed = time.time() - t0 + + # Normal map from model output + normal_map = images[0] if images else None + + # Depth map from Lpred + Lpred_img = Lpred[0].clamp(0, 1).cpu() + depth_map = F.to_pil_image(Lpred_img) + + status = f"Generated in {elapsed:.1f}s ({image.size[0]}x{image.size[1]}, single denoise)" + print(f"[gen] {status}") + return depth_map, normal_map, status + + +def main(): + import argparse + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command") + infer = sub.add_parser("infer") + infer.add_argument("-i", "--input", required=True) + infer.add_argument("-o", "--output-dir", default=".") + + args = parser.parse_args() + + if args.command == "infer": + from PIL import Image + depth, normal, status = generate(args.input) + depth.save(os.path.join(args.output_dir, "depth.png")) + if normal: + normal.save(os.path.join(args.output_dir, "normal.png")) + print(status) + return + + import gradio as gr + + with gr.Blocks(title="FE2E: Depth + Normal (CPU)") as demo: + gr.Markdown( + "**[FE2E](https://github.com/AMAP-ML/FE2E)** Depth + Normal estimation from a single image. " + "Single denoise step via Step1X-Edit DiT + LDRN LoRA. " + "CPU inference with dynamic INT8 quantization." + ) + with gr.Row(): + with gr.Column(): + input_img = gr.Image(label="Input Image", type="pil") + run_btn = gr.Button("Estimate Depth + Normal", variant="primary") + with gr.Column(): + depth_out = gr.Image(label="Depth Map", type="pil") + normal_out = gr.Image(label="Normal Map", type="pil") + status_out = gr.Textbox(label="Status", interactive=False) + + run_btn.click( + fn=generate, + inputs=[input_img], + outputs=[depth_out, normal_out, status_out], + concurrency_limit=1, + api_name="generate", + ) + + demo.queue(default_concurrency_limit=1) + demo.launch(server_name="0.0.0.0", server_port=7860, show_error=True, + mcp_server=True, ssr_mode=False) + + +if __name__ == "__main__": + main() diff --git a/infer/__init__.py b/infer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infer/alignment.py b/infer/alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..056b92524eb043755b599502298825e16a03e829 --- /dev/null +++ b/infer/alignment.py @@ -0,0 +1,70 @@ +import numpy as np +import torch + + +def align_depth_least_square( + gt_arr: np.ndarray, + pred_arr: np.ndarray, + valid_mask_arr: np.ndarray, + return_scale_shift=True, + max_resolution=None, +): + ori_shape = pred_arr.shape # input shape + + gt = gt_arr.squeeze() # [H, W] + pred = pred_arr.squeeze() + valid_mask = valid_mask_arr.squeeze() + + # Downsample + if max_resolution is not None: + scale_factor = np.min(max_resolution / np.array(ori_shape[-2:])) + if scale_factor < 1: + downscaler = torch.nn.Upsample(scale_factor=scale_factor, mode="nearest") + gt = downscaler(torch.as_tensor(gt).unsqueeze(0)).numpy() + pred = downscaler(torch.as_tensor(pred).unsqueeze(0)).numpy() + valid_mask = ( + downscaler(torch.as_tensor(valid_mask).unsqueeze(0).float()) + .bool() + .numpy() + ) + + assert ( + gt.shape == pred.shape == valid_mask.shape + ), f"{gt.shape}, {pred.shape}, {valid_mask.shape}" + + gt_masked = gt[valid_mask].reshape((-1, 1)) + pred_masked = pred[valid_mask].reshape((-1, 1)) + + # numpy solver + _ones = np.ones_like(pred_masked) + A = np.concatenate([pred_masked, _ones], axis=-1) + X = np.linalg.lstsq(A, gt_masked, rcond=None)[0] + scale, shift = X + + aligned_pred = pred_arr * scale + shift + + # restore dimensions + aligned_pred = aligned_pred.reshape(ori_shape) + + if return_scale_shift: + return aligned_pred, scale, shift + else: + return aligned_pred + + +# ******************** disparity space ******************** +def depth2disparity(depth, return_mask=False): + if isinstance(depth, torch.Tensor): + disparity = torch.zeros_like(depth) + elif isinstance(depth, np.ndarray): + disparity = np.zeros_like(depth) + non_negtive_mask = depth > 0 + disparity[non_negtive_mask] = 1.0 / depth[non_negtive_mask] + if return_mask: + return disparity, non_negtive_mask + else: + return disparity + + +def disparity2depth(disparity, **kwargs): + return depth2disparity(disparity, **kwargs) diff --git a/infer/configs/data_diode_all.yaml b/infer/configs/data_diode_all.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a5d7e15f5b6f6a9b28279b85d86dbfc1637654b --- /dev/null +++ b/infer/configs/data_diode_all.yaml @@ -0,0 +1,5 @@ +name: diode +disp_name: diode_val_all +dir: diode/diode_val.tar +filenames: infer/data_split/diode/diode_val_all_filename_list.txt +processing_res: 768 \ No newline at end of file diff --git a/infer/configs/data_eth3d.yaml b/infer/configs/data_eth3d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfe445d0fbf58d97bdcdebc0aa342f588bd7affe --- /dev/null +++ b/infer/configs/data_eth3d.yaml @@ -0,0 +1,7 @@ +name: eth3d +disp_name: eth3d_full +# dir: eth3d +dir: eth3d/eth3d.tar +filenames: infer/data_split/eth3d/eth3d_filename_list.txt +processing_res: 768 +alignment_max_res: 1024 diff --git a/infer/configs/data_kitti_eigen_test.yaml b/infer/configs/data_kitti_eigen_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c2ff3cd7fb3e5f3c99b833a4f25143820d367fb --- /dev/null +++ b/infer/configs/data_kitti_eigen_test.yaml @@ -0,0 +1,6 @@ +name: kitti +disp_name: kitti_eigen_test_full +dir: kitti/kitti_eigen_split_test.tar +filenames: infer/data_split/kitti/eigen_test_files_with_gt.txt +kitti_bm_crop: true +valid_mask_crop: eigen \ No newline at end of file diff --git a/infer/configs/data_nyu_test.yaml b/infer/configs/data_nyu_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2240f67d6850931dfea83efdb99e3c0ab3dd02c --- /dev/null +++ b/infer/configs/data_nyu_test.yaml @@ -0,0 +1,5 @@ +name: nyu_v2 +disp_name: nyu_test_full +dir: nyudepth/nyu_labeled_extracted.tar +filenames: infer/data_split/nyu/filename_list_test.txt +eigen_valid_mask: true \ No newline at end of file diff --git a/infer/configs/data_scannet_val.yaml b/infer/configs/data_scannet_val.yaml new file mode 100644 index 0000000000000000000000000000000000000000..647a02fcebe36dd15cf9959e7d3c0d6bf0a4c25f --- /dev/null +++ b/infer/configs/data_scannet_val.yaml @@ -0,0 +1,5 @@ +name: scannet +disp_name: scannet_val_800_1 +# dir: scannet +dir: scannet/scannet_val_sampled_800_1.tar +filenames: infer/data_split/scannet/scannet_val_sampled_list_800_1.txt \ No newline at end of file diff --git a/infer/dataset/__init__.py b/infer/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a99e8721b8625e9744853cf4ebbc6894545330b --- /dev/null +++ b/infer/dataset/__init__.py @@ -0,0 +1,49 @@ +import os + +from .base_depth_dataset import BaseDepthDataset, get_pred_name, DatasetMode # noqa: F401 +from .diode_dataset import DIODEDataset +from .eth3d_dataset import ETH3DDataset +from .kitti_dataset import KITTIDataset +from .nyu_dataset import NYUDataset +from .scannet_dataset import ScanNetDataset + + +dataset_name_class_dict = { + "nyu_v2": NYUDataset, + "kitti": KITTIDataset, + "eth3d": ETH3DDataset, + "diode": DIODEDataset, + "scannet": ScanNetDataset, +} + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def _resolve_split_file(path: str) -> str: + if os.path.isabs(path) or os.path.exists(path): + return path + + candidate = os.path.join(REPO_ROOT, path) + if os.path.exists(candidate): + return candidate + + return path + + +def get_dataset(cfg_data_split, base_data_dir: str, mode: DatasetMode, prompt_type="query", **kwargs) -> BaseDepthDataset: + if cfg_data_split.name in dataset_name_class_dict.keys(): + dataset_class = dataset_name_class_dict[cfg_data_split.name] + filename_ls_path = cfg_data_split.filenames if not prompt_type == "full" else (cfg_data_split.filenames).replace(".txt", "_wc.txt") + filename_ls_path = _resolve_split_file(filename_ls_path) + dataset = dataset_class( + mode=mode, + filename_ls_path=filename_ls_path, + dataset_dir=os.path.join(base_data_dir, cfg_data_split.dir), + **cfg_data_split, + prompt_type=prompt_type, + **kwargs, + ) + else: + raise NotImplementedError + + return dataset diff --git a/infer/dataset/base_depth_dataset.py b/infer/dataset/base_depth_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a5b4cbbd43e8c65f9d160bc26bba81071c300036 --- /dev/null +++ b/infer/dataset/base_depth_dataset.py @@ -0,0 +1,257 @@ +import io +import os +import random +import tarfile +from enum import Enum + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import Dataset +from torchvision.transforms import InterpolationMode, Resize +import torchvision.transforms.functional as F + +class DatasetMode(Enum): + RGB_ONLY = "rgb_only" + EVAL = "evaluate" + TRAIN = "train" + + +def read_image_from_tar(tar_obj, img_rel_path): + image = tar_obj.extractfile("./" + img_rel_path) + image = image.read() + image = Image.open(io.BytesIO(image)) + + +class BaseDepthDataset(Dataset): + + def __init__( + self, mode: DatasetMode, filename_ls_path: str, dataset_dir: str, disp_name: str, min_depth, max_depth, has_filled_depth, name_mode, depth_transform=None, augmentation_args: dict = None, resize_to_hw=None, + move_invalid_to_far_plane: bool = True, rgb_transform=None, prompt_type="query", **kwargs, + ) -> None: + super().__init__() + self.mode = mode + + self.filename_ls_path = filename_ls_path + self.dataset_dir = dataset_dir + self.disp_name = disp_name + self.has_filled_depth = has_filled_depth + self.name_mode: DepthFileNameMode = name_mode + self.min_depth = min_depth + self.max_depth = max_depth + + self.depth_transform = depth_transform + self.augm_args = augmentation_args + self.resize_to_hw = resize_to_hw + self.prompt_type = prompt_type + # 设置默认的rgb_transform函数 + if rgb_transform is None: + self.rgb_transform = self._default_rgb_transform + else: + self.rgb_transform = rgb_transform + self.move_invalid_to_far_plane = move_invalid_to_far_plane + + # Load filenames + with open(self.filename_ls_path, "r") as f: + self.filenames = [s.split() for s in f.readlines()] # [['rgb.png', 'depth.tif'], [], ...] + + # Tar dataset + self.tar_obj = None + self.tar_obj_pid = None + self.is_tar = (True if os.path.isfile(dataset_dir) and tarfile.is_tarfile(dataset_dir) else False) + + def __len__(self): + return len(self.filenames) + + def __getitem__(self, index): + rasters, other = self._get_data_item(index) + if DatasetMode.TRAIN == self.mode: + rasters = self._training_preprocess(rasters) + # merge + outputs = rasters + outputs.update(other) + return outputs + + def _get_data_item(self, index): + rgb_rel_path, depth_rel_path, filled_rel_path, prompt = self._get_data_path(index=index) + + rasters = {} + + # RGB data + rasters.update(self._load_rgb_data(rgb_rel_path=rgb_rel_path)) + + # Depth data + if DatasetMode.RGB_ONLY != self.mode: + # load data + depth_data = self._load_depth_data(depth_rel_path=depth_rel_path, filled_rel_path=filled_rel_path) + rasters.update(depth_data) + # valid mask + rasters["valid_mask_raw"] = self._get_valid_mask(rasters["depth_raw_linear"]).clone() + rasters["valid_mask_filled"] = self._get_valid_mask(rasters["depth_filled_linear"]).clone() + + + other = {"index": index, "rgb_relative_path": rgb_rel_path, "prompt": prompt} + + return rasters, other + + def _load_rgb_data(self, rgb_rel_path): + # Read RGB data + _, rgb = self._read_image(rgb_rel_path) + rgb = self.input_process_image(rgb) + outputs = {"rgb": rgb} + return outputs + + def input_process_image(self, image): + if isinstance(image, np.ndarray): + image = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0 + + return image + elif isinstance(image, Image.Image): + image = F.to_tensor(image.convert("RGB")) + + return image + elif isinstance(image, torch.Tensor): + return image + elif isinstance(image, str): + image = F.to_tensor(Image.open(image).convert("RGB")) + + return image + return image + + def _load_depth_data(self, depth_rel_path, filled_rel_path): + # Read depth data + outputs = {} + depth_raw = self._read_depth_file(depth_rel_path).squeeze() + depth_raw_linear = torch.from_numpy(depth_raw).float().unsqueeze(0) # [1, H, W] + outputs["depth_raw_linear"] = depth_raw_linear.clone() + + if self.has_filled_depth: + depth_filled = self._read_depth_file(filled_rel_path).squeeze() + depth_filled_linear = torch.from_numpy(depth_filled).float().unsqueeze(0) + outputs["depth_filled_linear"] = depth_filled_linear + else: + outputs["depth_filled_linear"] = depth_raw_linear.clone() + + return outputs + + def _get_data_path(self, index): + filename_line = self.filenames[index] + rgb_rel_path = filename_line[0] + depth_rel_path, filled_rel_path = None, None + if DatasetMode.RGB_ONLY != self.mode: + depth_rel_path = filename_line[1] + if self.has_filled_depth: + filled_rel_path = filename_line[2] + + if self.prompt_type == "full": + if filename_line[2][0] in ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]: + prompt = ' '.join(filename_line[2:]) + else: + prompt = ' '.join(filename_line[3:]) + else: + prompt = 1 + return rgb_rel_path, depth_rel_path, filled_rel_path, prompt + + def _read_image(self, img_rel_path): + if self.is_tar: + tar_obj = self._ensure_tar_obj() + image = tar_obj.extractfile("./" + img_rel_path) + image = image.read() + image = Image.open(io.BytesIO(image)) + else: + img_path = os.path.join(self.dataset_dir, img_rel_path) + image = Image.open(img_path) + image_arr = np.asarray(image) + return image_arr, image + + def _read_depth_file(self, rel_path): + depth_in, _ = self._read_image(rel_path) + # Replace code below to decode depth according to dataset definition + depth_decoded = depth_in + + return depth_decoded + + def _get_valid_mask(self, depth: torch.Tensor): + valid_mask = torch.logical_and((depth > self.min_depth), (depth < self.max_depth)).bool() + return valid_mask + + def _training_preprocess(self, rasters): + # Augmentation + if self.augm_args is not None: + rasters = self._augment_data(rasters) + + # Normalization + rasters["depth_raw_norm"] = self.depth_transform(rasters["depth_raw_linear"], rasters["valid_mask_raw"]).clone() + rasters["depth_filled_norm"] = self.depth_transform(rasters["depth_filled_linear"], rasters["valid_mask_filled"]).clone() + + # Set invalid pixel to far plane + if self.move_invalid_to_far_plane: + if self.depth_transform.far_plane_at_max: + rasters["depth_filled_norm"][~rasters["valid_mask_filled"]] = self.depth_transform.norm_max + else: + rasters["depth_filled_norm"][~rasters["valid_mask_filled"]] = self.depth_transform.norm_min + + # Resize + if self.resize_to_hw is not None: + resize_transform = Resize(size=self.resize_to_hw, interpolation=InterpolationMode.NEAREST_EXACT) + rasters = {k: resize_transform(v) for k, v in rasters.items()} + + return rasters + + def _augment_data(self, rasters_dict): + # lr flipping + lr_flip_p = self.augm_args.lr_flip_p + if random.random() < lr_flip_p: + rasters_dict = {k: v.flip(-1) for k, v in rasters_dict.items()} + + return rasters_dict + + def __del__(self): + if self.tar_obj is not None: + self.tar_obj.close() + self.tar_obj = None + self.tar_obj_pid = None + + def _default_rgb_transform(self, x): + """默认的RGB变换函数: [0, 255] -> [-1, 1]""" + return x / 255.0 * 2 - 1 + + def _ensure_tar_obj(self): + """Ensure each process owns its own tar handle to avoid cross-process FD issues.""" + if not self.is_tar: + return None + current_pid = os.getpid() + if self.tar_obj is None or self.tar_obj_pid != current_pid: + if self.tar_obj is not None: + try: + self.tar_obj.close() + except Exception: + pass + self.tar_obj = tarfile.open(self.dataset_dir) + self.tar_obj_pid = current_pid + return self.tar_obj + + +# Prediction file naming modes +class DepthFileNameMode(Enum): + id = 1 # id.png + rgb_id = 2 # rgb_id.png + i_d_rgb = 3 # i_d_1_rgb.png + rgb_i_d = 4 + + +def get_pred_name(rgb_basename, name_mode, suffix=".png"): + if DepthFileNameMode.rgb_id == name_mode: + pred_basename = "pred_" + rgb_basename.split("_")[1] + elif DepthFileNameMode.i_d_rgb == name_mode: + pred_basename = rgb_basename.replace("_rgb.", "_pred.") + elif DepthFileNameMode.id == name_mode: + pred_basename = "pred_" + rgb_basename + elif DepthFileNameMode.rgb_i_d == name_mode: + pred_basename = "pred_" + "_".join(rgb_basename.split("_")[1:]) + else: + raise NotImplementedError + # change suffix + pred_basename = os.path.splitext(pred_basename)[0] + suffix + + return pred_basename diff --git a/infer/dataset/diode_dataset.py b/infer/dataset/diode_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f764983213c8d9b5f5f0708248f85a2ff10ddc8b --- /dev/null +++ b/infer/dataset/diode_dataset.py @@ -0,0 +1,71 @@ +# Author: Bingxin Ke +# Last modified: 2024-02-26 + +import os +import tarfile +from io import BytesIO + +import numpy as np +import torch + +from .base_depth_dataset import BaseDepthDataset, DepthFileNameMode, DatasetMode + + +class DIODEDataset(BaseDepthDataset): + def __init__( + self, + **kwargs, + ) -> None: + super().__init__( + # DIODE data parameter + min_depth=0.6, + max_depth=350, + has_filled_depth=False, + name_mode=DepthFileNameMode.id, + **kwargs, + ) + + def _read_npy_file(self, rel_path): + if self.is_tar: + if self.tar_obj is None: + self.tar_obj = tarfile.open(self.dataset_dir) + fileobj = self.tar_obj.extractfile("./" + rel_path) + npy_path_or_content = BytesIO(fileobj.read()) + else: + npy_path_or_content = os.path.join(self.dataset_dir, rel_path) + data = np.load(npy_path_or_content).squeeze()[np.newaxis, :, :] + return data + + def _read_depth_file(self, rel_path): + depth = self._read_npy_file(rel_path) + return depth + + def _get_data_path(self, index): + return self.filenames[index][0], self.filenames[index][1], self.filenames[index][2], self.filenames[index][3:] if len(self.filenames[index]) > 3 else 1 + + def _get_data_item(self, index): + + rgb_rel_path, depth_rel_path, mask_rel_path, prompt = self._get_data_path(index=index) + + rasters = {} + + # RGB data + rasters.update(self._load_rgb_data(rgb_rel_path=rgb_rel_path)) + + # Depth data + if DatasetMode.RGB_ONLY != self.mode: + # load data + depth_data = self._load_depth_data( + depth_rel_path=depth_rel_path, filled_rel_path=None + ) + rasters.update(depth_data) + + # valid mask + mask = self._read_npy_file(mask_rel_path).astype(bool) + mask = torch.from_numpy(mask).bool() + rasters["valid_mask_raw"] = mask.clone() + rasters["valid_mask_filled"] = mask.clone() + + other = {"index": index, "rgb_relative_path": rgb_rel_path, "prompt": prompt} + + return rasters, other diff --git a/infer/dataset/drivingstereo_dataset.py b/infer/dataset/drivingstereo_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..83db08ddfbc641ed7307c028e1f90c9a717d3e76 --- /dev/null +++ b/infer/dataset/drivingstereo_dataset.py @@ -0,0 +1,32 @@ +import os +import json +import numpy as np +import PIL.Image as pil + +from .mono_dataset import MonoDataset + + +class DrivingStereoDataset(MonoDataset): + RAW_HEIGHT = 800 + RAW_WIDTH = 1762 + + def __init__(self, *args, **kwargs): + super(DrivingStereoDataset, self).__init__(*args, **kwargs) + self.forename = {"rainy": "2018-08-17-09-45-58_2018-08-17-10-", "foggy": "2018-10-25-07-37-26_2018-10-25-", "sunny": "2018-10-19-09-30-39_2018-10-19-", "cloudy": "2018-10-31-06-55-01_2018-10-31-"} + + def get_color(self, weather, name, do_flip): + path, name = self.get_image_path(weather, name) + color = self.loader(path) + + return color, name + + def get_image_path(self, weather, frame_name): + folder = "left-image-full-size" + image_path = os.path.join(self.opts.data_path, weather, folder, frame_name) + image_name = os.path.join(weather, folder, frame_name) + if self.opts.debug >= 3: + print(image_name) + return image_path, image_name + + def index_to_name(self, weather, index): + return self.forename[weather] + self.filenames[index] + ".png" diff --git a/infer/dataset/eth3d_dataset.py b/infer/dataset/eth3d_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..75c1b809886e466207c2574992e4dba4ab421097 --- /dev/null +++ b/infer/dataset/eth3d_dataset.py @@ -0,0 +1,45 @@ +# Author: Bingxin Ke +# Last modified: 2024-02-08 + +import torch +import tarfile +import os +import numpy as np + +from .base_depth_dataset import BaseDepthDataset, DepthFileNameMode + + +class ETH3DDataset(BaseDepthDataset): + HEIGHT, WIDTH = 4032, 6048 + + def __init__( + self, + **kwargs, + ) -> None: + super().__init__( + # ETH3D data parameter + min_depth=1e-5, + max_depth=torch.inf, + has_filled_depth=False, + name_mode=DepthFileNameMode.id, + **kwargs, + ) + + def _read_depth_file(self, rel_path): + # Read special binary data: https://www.eth3d.net/documentation#format-of-multi-view-data-image-formats + if self.is_tar: + tar_obj = self._ensure_tar_obj() + binary_data = tar_obj.extractfile("./" + rel_path) + binary_data = binary_data.read() + + else: + depth_path = os.path.join(self.dataset_dir, rel_path) + with open(depth_path, "rb") as file: + binary_data = file.read() + # Convert the binary data to a numpy array of 32-bit floats + depth_decoded = np.frombuffer(binary_data, dtype=np.float32).copy() + + depth_decoded[depth_decoded == torch.inf] = 0.0 + + depth_decoded = depth_decoded.reshape((self.HEIGHT, self.WIDTH)) + return depth_decoded diff --git a/infer/dataset/kitti_dataset.py b/infer/dataset/kitti_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..74a95362a5e4af5d3d386a089e8215fc5b2204d0 --- /dev/null +++ b/infer/dataset/kitti_dataset.py @@ -0,0 +1,105 @@ +# Author: Bingxin Ke +# Last modified: 2024-02-08 + +import torch + +from .base_depth_dataset import BaseDepthDataset, DepthFileNameMode + + +class KITTIDataset(BaseDepthDataset): + def __init__( + self, + kitti_bm_crop, # Crop to KITTI benchmark size + valid_mask_crop, # Evaluation mask. [None, garg or eigen] + **kwargs, + ) -> None: + super().__init__( + # KITTI data parameter + min_depth=1e-5, + max_depth=80, + has_filled_depth=False, + name_mode=DepthFileNameMode.id, + **kwargs, + ) + self.kitti_bm_crop = kitti_bm_crop + self.valid_mask_crop = valid_mask_crop + assert self.valid_mask_crop in [ + None, + "garg", # set evaluation mask according to Garg ECCV16 + "eigen", # set evaluation mask according to Eigen NIPS14 + ], f"Unknown crop type: {self.valid_mask_crop}" + + # Filter out empty depth + self.filenames = [f for f in self.filenames if "None" != f[1]] + + def _read_depth_file(self, rel_path): + depth_in,_ = self._read_image(rel_path) + # Decode KITTI depth + depth_decoded = depth_in / 256.0 + return depth_decoded + + def _load_rgb_data(self, rgb_rel_path): + rgb_data = super()._load_rgb_data(rgb_rel_path) + if self.kitti_bm_crop: + rgb_data = {k: self.kitti_benchmark_crop(v) for k, v in rgb_data.items()} + return rgb_data + + def _load_depth_data(self, depth_rel_path, filled_rel_path): + depth_data = super()._load_depth_data(depth_rel_path, filled_rel_path) + if self.kitti_bm_crop: + depth_data = { + k: self.kitti_benchmark_crop(v) for k, v in depth_data.items() + } + return depth_data + + @staticmethod + def kitti_benchmark_crop(input_img): + """ + Crop images to KITTI benchmark size + Args: + `input_img` (torch.Tensor): Input image to be cropped. + + Returns: + torch.Tensor:Cropped image. + """ + KB_CROP_HEIGHT = 352 + KB_CROP_WIDTH = 1216 + + height, width = input_img.shape[-2:] + top_margin = int(height - KB_CROP_HEIGHT) + left_margin = int((width - KB_CROP_WIDTH) / 2) + if 2 == len(input_img.shape): + out = input_img[ + top_margin: top_margin + KB_CROP_HEIGHT, + left_margin: left_margin + KB_CROP_WIDTH, + ] + elif 3 == len(input_img.shape): + out = input_img[ + :, + top_margin: top_margin + KB_CROP_HEIGHT, + left_margin: left_margin + KB_CROP_WIDTH, + ] + return out + + def _get_valid_mask(self, depth: torch.Tensor): + # reference: https://github.com/cleinc/bts/blob/master/pytorch/bts_eval.py + valid_mask = super()._get_valid_mask(depth) # [1, H, W] + + if self.valid_mask_crop is not None: + eval_mask = torch.zeros_like(valid_mask.squeeze()).bool() + gt_height, gt_width = eval_mask.shape + + if "garg" == self.valid_mask_crop: + eval_mask[ + int(0.40810811 * gt_height): int(0.99189189 * gt_height), + int(0.03594771 * gt_width): int(0.96405229 * gt_width), + ] = 1 + elif "eigen" == self.valid_mask_crop: + eval_mask[ + int(0.3324324 * gt_height): int(0.91351351 * gt_height), + int(0.0359477 * gt_width): int(0.96405229 * gt_width), + ] = 1 + + eval_mask.reshape(valid_mask.shape) + valid_mask = torch.logical_and(valid_mask, eval_mask) + return valid_mask diff --git a/infer/dataset/nyu_dataset.py b/infer/dataset/nyu_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3dbcee3cf68703a917a6248aa097d503ec17a844 --- /dev/null +++ b/infer/dataset/nyu_dataset.py @@ -0,0 +1,43 @@ +# Author: Bingxin Ke +# Last modified: 2024-02-08 + + +import torch + +from .base_depth_dataset import BaseDepthDataset, DepthFileNameMode + + +class NYUDataset(BaseDepthDataset): + def __init__( + self, + eigen_valid_mask: bool, + **kwargs, + ) -> None: + super().__init__( + # NYUv2 dataset parameter + min_depth=1e-3, + max_depth=10.0, + has_filled_depth=True, + name_mode=DepthFileNameMode.rgb_id, + **kwargs, + ) + + self.eigen_valid_mask = eigen_valid_mask + + def _read_depth_file(self, rel_path): + depth_in,_ = self._read_image(rel_path) + # Decode NYU depth + depth_decoded = depth_in / 1000.0 + return depth_decoded + + def _get_valid_mask(self, depth: torch.Tensor): + valid_mask = super()._get_valid_mask(depth) + + # Eigen crop for evaluation + if self.eigen_valid_mask: + eval_mask = torch.zeros_like(valid_mask.squeeze()).bool() + eval_mask[45:471, 41:601] = 1 + eval_mask.reshape(valid_mask.shape) + valid_mask = torch.logical_and(valid_mask, eval_mask) + + return valid_mask diff --git a/infer/dataset/scannet_dataset.py b/infer/dataset/scannet_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..6bdfbb9a2f9cd0abc4a3e253f0618f6bf9e07436 --- /dev/null +++ b/infer/dataset/scannet_dataset.py @@ -0,0 +1,25 @@ +# Author: Bingxin Ke +# Last modified: 2024-02-08 + +from .base_depth_dataset import BaseDepthDataset, DepthFileNameMode + + +class ScanNetDataset(BaseDepthDataset): + def __init__( + self, + **kwargs, + ) -> None: + super().__init__( + # ScanNet data parameter + min_depth=1e-3, + max_depth=10, + has_filled_depth=False, + name_mode=DepthFileNameMode.id, + **kwargs, + ) + + def _read_depth_file(self, rel_path): + depth_in,_ = self._read_image(rel_path) + # Decode ScanNet depth + depth_decoded = depth_in / 1000.0 + return depth_decoded diff --git a/infer/dataset_normal/__init__.py b/infer/dataset_normal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fa59c8f842505001baa3f69d812a5afd2baf5c35 --- /dev/null +++ b/infer/dataset_normal/__init__.py @@ -0,0 +1,27 @@ +""" data sample +""" +class Sample(): + def __init__(self, img=None, + depth=None, depth_mask=None, + normal=None, normal_mask=None, + intrins=None, flipped=False, + dataset_name='dataset', scene_name='scene', img_name='img', + info={}): + + self.img = img # input image + + self.depth = depth # depth - GT + self.depth_mask = depth_mask # depth - valid_mask + + self.normal = normal # surface normals - GT + self.normal_mask = normal_mask # surface normals - valid_mask + + self.intrins = intrins # camera intrinsics + self.flipped = flipped # True when the image is flipped during augmentation + + self.dataset_name = dataset_name + self.scene_name = scene_name + self.img_name = img_name + + # other info (this is a dict containing any additional information) + self.info = info \ No newline at end of file diff --git a/infer/dataset_normal/aug_basic.py b/infer/dataset_normal/aug_basic.py new file mode 100644 index 0000000000000000000000000000000000000000..1b92229f5bbadeefe2b3722f7cae5b955fb41f2c --- /dev/null +++ b/infer/dataset_normal/aug_basic.py @@ -0,0 +1,239 @@ +""" basic augmentations +""" +import random +import numpy as np + +import torch +from torchvision import transforms +import torch.nn.functional as F +import torchvision.transforms.functional as TF + +import logging +logger = logging.getLogger('root') + + +def resize(sample, new_H, new_W): + _, orig_H, orig_W = sample.img.shape + sample.img = F.interpolate(sample.img.unsqueeze(0), size=(new_H, new_W), mode='bilinear', align_corners=False, antialias=True).squeeze(0) + if sample.depth is not None: + sample.depth = F.interpolate(sample.depth.unsqueeze(0), size=(new_H, new_W), mode='nearest').squeeze(0) + if sample.depth_mask is not None: + sample.depth_mask = F.interpolate(sample.depth_mask.unsqueeze(0).float(), size=(new_H, new_W), mode='nearest').squeeze(0) > 0.5 + if sample.normal is not None: + sample.normal = F.interpolate(sample.normal.unsqueeze(0), size=(new_H, new_W), mode='nearest').squeeze(0) + if sample.normal_mask is not None: + sample.normal_mask = F.interpolate(sample.normal_mask.unsqueeze(0).float(), size=(new_H, new_W), mode='nearest').squeeze(0) > 0.5 + if sample.intrins is not None: + # NOTE: top-left is (0,0) + sample.intrins[0, 0] = sample.intrins[0, 0] * (new_W / orig_W) # fx + sample.intrins[1, 1] = sample.intrins[1, 1] * (new_H / orig_H) # fy + sample.intrins[0, 2] = (sample.intrins[0, 2] + 0.5) * (new_W / orig_W) - 0.5 # cx + sample.intrins[1, 2] = (sample.intrins[1, 2] + 0.5) * (new_H / orig_H) - 0.5 # cy + return sample + + +def pad(sample, lrtb): + l, r, t, b = lrtb + sample.img = F.pad(sample.img, (l, r, t, b), mode="constant", value=0) + if sample.depth is not None: + sample.depth = F.pad(sample.depth, (l, r, t, b), mode="constant", value=0) + if sample.depth_mask is not None: + sample.depth_mask = F.pad(sample.depth_mask, (l, r, t, b), mode="constant", value=False) + if sample.normal is not None: + sample.normal = F.pad(sample.normal, (l, r, t, b), mode="constant", value=0) + if sample.normal_mask is not None: + sample.normal_mask = F.pad(sample.normal_mask, (l, r, t, b), mode="constant", value=False) + if sample.intrins is not None: + sample.intrins[0, 2] = sample.intrins[0, 2] + l + sample.intrins[1, 2] = sample.intrins[1, 2] + t + return sample + + +def crop(sample, y, H, x, W): + sample.img = sample.img[:, y:y+H, x:x+W] + if sample.depth is not None: + sample.depth = sample.depth[:, y:y+H, x:x+W] + if sample.depth_mask is not None: + sample.depth_mask = sample.depth_mask[:, y:y+H, x:x+W] + if sample.normal is not None: + sample.normal = sample.normal[:, y:y+H, x:x+W] + if sample.normal_mask is not None: + sample.normal_mask = sample.normal_mask[:, y:y+H, x:x+W] + if sample.intrins is not None: + sample.intrins[0, 2] = sample.intrins[0, 2] - x + sample.intrins[1, 2] = sample.intrins[1, 2] - y + return sample + + +class ToTensor(): + """ numpy arrays to torch tensors + """ + def __call__(self, sample): + sample.img = torch.from_numpy(sample.img).permute(2, 0, 1) # (3, H, W) + if sample.depth is not None: + sample.depth = torch.from_numpy(sample.depth).permute(2, 0, 1) # (1, H, W) + if sample.depth_mask is not None: + sample.depth_mask = torch.from_numpy(sample.depth_mask).permute(2, 0, 1) # (1, H, W) + if sample.normal is not None: + sample.normal = torch.from_numpy(sample.normal).permute(2, 0, 1) # (3, H, W) + if sample.normal_mask is not None: + sample.normal_mask = torch.from_numpy(sample.normal_mask).permute(2, 0, 1) # (1, H, W) + if sample.intrins is not None: + sample.intrins = torch.from_numpy(sample.intrins) # (3, 3) + return sample + + +class RandomIntrins(): + """ randomize intrinsics + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __call__(self, sample): + assert 'crop_H' in sample.info.keys() + assert 'crop_W' in sample.info.keys() + crop_H = sample.info['crop_H'] + crop_W = sample.info['crop_W'] + + # height-based resizing + _, orig_H, orig_W = sample.img.shape + new_H = random.randrange(min(orig_H, crop_H), max(orig_H, crop_H)+1) + new_W = round((new_H / orig_H) * orig_W) + sample = resize(sample, new_H=new_H, new_W=new_W) + + # pad if necessary + orig_H, orig_W = sample.img.shape[1], sample.img.shape[2] + l, r, t, b = 0, 0, 0, 0 + if crop_H > orig_H: + t = b = crop_H - orig_H + if crop_W > orig_W: + l = r = crop_W - orig_W + sample = pad(sample, (l, r, t, b)) + + # crop + assert sample.img.shape[1] >= crop_H + assert sample.img.shape[2] >= crop_W + x = random.randint(0, sample.img.shape[2] - crop_W) + y = random.randint(0, sample.img.shape[1] - crop_H) + sample = crop(sample, y=y, H=crop_H, x=x, W=crop_W) + + return sample + + +class Resize(): + """ resize to (H, W) + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __init__(self, H=480, W=640): + self.H = H + self.W = W + + def __call__(self, sample): + return resize(sample, new_H=self.H, new_W=self.W) + + +class RandomCrop(): + """ random crop + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __init__(self, H=416, W=544): + self.H = H + self.W = W + + def __call__(self, sample): + assert sample.img.shape[1] >= self.H + assert sample.img.shape[2] >= self.W + x = random.randint(0, sample.img.shape[2] - self.W) + y = random.randint(0, sample.img.shape[1] - self.H) + return crop(sample, y=y, H=self.H, x=x, W=self.W) + + +class NyuCrop(): + """ crop image border for NYUv2 images + W = 43:608 / H = 45:472 + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __call__(self, sample): + return crop(sample, y=45, H=472-45, x=43, W=608-43) + + +class HorizontalFlip(): + """ random horizontal flipping + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __init__(self, p=0.5): + self.p = p + + def __call__(self, sample): + if random.random() < self.p: + sample.img = TF.hflip(sample.img) + if sample.depth is not None: + sample.depth = TF.hflip(sample.depth) + if sample.depth_mask is not None: + sample.depth_mask = TF.hflip(sample.depth_mask) + if sample.normal is not None: + sample.normal = TF.hflip(sample.normal) + sample.normal[0, :, :] = -sample.normal[0, :, :] + if sample.normal_mask is not None: + sample.normal_mask = TF.hflip(sample.normal_mask) + if sample.intrins is not None: + # NOTE: top-left is (0,0) + _, H, W = sample.img.shape + sample.intrins[0, 2] = sample.intrins[0, 2] + 0.5 # top-left is (0.5, 0.5) + sample.intrins[0, 2] = W - sample.intrins[0, 2] + sample.intrins[0, 2] = sample.intrins[0, 2] - 0.5 # top-left is (0, 0) + sample.flipped = True + return sample + + +class ColorAugmentation(): + """ color augmentation + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __init__(self, gamma_range=(0.9, 1.1), + brightness_range=(0.75, 1.25), + color_range=(0.9, 1.1), + p=0.5): + self.gamma_range = gamma_range + self.brightness_range = brightness_range + self.color_range = color_range + self.p = p + + def __call__(self, sample): + if random.random() < self.p: + # gamma augmentation + gamma = random.uniform(*self.gamma_range) + sample.img = sample.img ** gamma + + # brightness augmentation + brightness = random.uniform(*self.brightness_range) + sample.img = sample.img * brightness + + # color augmentation + colors = np.random.uniform(*self.color_range, size=3).astype(np.float32) + colors = torch.from_numpy(colors).view(3, 1, 1) + sample.img = sample.img * colors + + # clip + sample.img = torch.clip(sample.img, 0, 1) + + return sample + + +class Normalize(): + """ mean & std: for image normalization + sample.img is a torch tensor of shape (3, H, W), normalized to [0, 1] + """ + def __init__(self, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)): + self.normalize = transforms.Normalize(mean=mean, std=std) + + def __call__(self, sample): + sample.img = self.normalize(torch.clip(sample.img, min=0.0, max=1.0)) + return sample + + +class ToDict(): + def __call__(self, sample): + data_dict = {} + for k, v in vars(sample).items(): + if v is not None: + data_dict[k] = v + return data_dict diff --git a/infer/dataset_normal/hypersim/__init__.py b/infer/dataset_normal/hypersim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b68f395622b4007ea267b254a474843f570e2535 --- /dev/null +++ b/infer/dataset_normal/hypersim/__init__.py @@ -0,0 +1,65 @@ +""" Get samples from Hypersim dataset + Based on vkitti implementation +""" +import os +import cv2 +import numpy as np + +from infer.dataset_normal import Sample + + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "ai_001_001/rgb_cam_00_fr0000.png" + scene_name = sample_path.split('/')[0] + img_filename = sample_path.split('/')[1] + + # Extract frame number from filename like "rgb_cam_00_fr0000.png" + frame_num = img_filename.split('_fr')[1].split('.')[0] + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'hypersim') + img_path = os.path.join(dataset_path, sample_path) + + # Build corresponding normal path + normal_filename = f'depth_plane_cam_00_fr{frame_num}_1024x0768_normal_decoded_normal.png' + normal_path = os.path.join(dataset_path, scene_name, normal_filename) + + assert os.path.exists(img_path), f"Image not found: {img_path}" + assert os.path.exists(normal_path), f"Normal not found: {normal_path}" + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + + # read normal (H, W, 3) + normal = cv2.cvtColor(cv2.imread(normal_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + normal_mask = np.sum(normal, axis=2, keepdims=True) > 0 + normal = (normal.astype(np.float32) / 255.0) * 2.0 - 1.0 + + # Create default intrinsics since hypersim doesn't provide them + # Using typical values for 1024x768 resolution + H, W = img.shape[:2] + fx = fy = 0.8 * W # Typical focal length assumption + cx = W / 2.0 + cy = H / 2.0 + intrins = np.array([ + [fx, 0, cx], + [0, fy, cy], + [0, 0, 1] + ], dtype=np.float32) + + # Extract img_name for compatibility + img_name = img_filename.replace('.png', '') + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='hypersim', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample \ No newline at end of file diff --git a/infer/dataset_normal/hypersim/split/hypersim.txt b/infer/dataset_normal/hypersim/split/hypersim.txt new file mode 100644 index 0000000000000000000000000000000000000000..7626459795b37d209a969caca8cb63edd2789af3 --- /dev/null +++ b/infer/dataset_normal/hypersim/split/hypersim.txt @@ -0,0 +1,98 @@ +ai_001_001/rgb_cam_00_fr0000.png +ai_001_001/rgb_cam_00_fr0001.png +ai_001_001/rgb_cam_00_fr0002.png +ai_001_001/rgb_cam_00_fr0003.png +ai_001_001/rgb_cam_00_fr0004.png +ai_001_001/rgb_cam_00_fr0005.png +ai_001_001/rgb_cam_00_fr0006.png +ai_001_001/rgb_cam_00_fr0007.png +ai_001_001/rgb_cam_00_fr0008.png +ai_001_001/rgb_cam_00_fr0009.png +ai_001_001/rgb_cam_00_fr0010.png +ai_001_001/rgb_cam_00_fr0011.png +ai_001_001/rgb_cam_00_fr0012.png +ai_001_001/rgb_cam_00_fr0013.png +ai_001_001/rgb_cam_00_fr0014.png +ai_001_001/rgb_cam_00_fr0015.png +ai_001_001/rgb_cam_00_fr0016.png +ai_001_001/rgb_cam_00_fr0017.png +ai_001_001/rgb_cam_00_fr0018.png +ai_001_001/rgb_cam_00_fr0019.png +ai_001_001/rgb_cam_00_fr0020.png +ai_001_001/rgb_cam_00_fr0021.png +ai_001_001/rgb_cam_00_fr0022.png +ai_001_001/rgb_cam_00_fr0023.png +ai_001_001/rgb_cam_00_fr0024.png +ai_001_001/rgb_cam_00_fr0025.png +ai_001_001/rgb_cam_00_fr0026.png +ai_001_001/rgb_cam_00_fr0027.png +ai_001_001/rgb_cam_00_fr0028.png +ai_001_001/rgb_cam_00_fr0029.png +ai_001_001/rgb_cam_00_fr0030.png +ai_001_001/rgb_cam_00_fr0031.png +ai_001_001/rgb_cam_00_fr0032.png +ai_001_001/rgb_cam_00_fr0033.png +ai_001_001/rgb_cam_00_fr0034.png +ai_001_001/rgb_cam_00_fr0035.png +ai_001_001/rgb_cam_00_fr0037.png +ai_001_001/rgb_cam_00_fr0038.png +ai_001_001/rgb_cam_00_fr0039.png +ai_001_001/rgb_cam_00_fr0040.png +ai_001_001/rgb_cam_00_fr0041.png +ai_001_001/rgb_cam_00_fr0042.png +ai_001_001/rgb_cam_00_fr0043.png +ai_001_001/rgb_cam_00_fr0044.png +ai_001_001/rgb_cam_00_fr0045.png +ai_001_001/rgb_cam_00_fr0046.png +ai_001_001/rgb_cam_00_fr0047.png +ai_001_001/rgb_cam_00_fr0048.png +ai_001_001/rgb_cam_00_fr0049.png +ai_001_001/rgb_cam_00_fr0050.png +ai_001_001/rgb_cam_00_fr0051.png +ai_001_001/rgb_cam_00_fr0052.png +ai_001_001/rgb_cam_00_fr0053.png +ai_001_001/rgb_cam_00_fr0054.png +ai_001_001/rgb_cam_00_fr0055.png +ai_001_001/rgb_cam_00_fr0056.png +ai_001_001/rgb_cam_00_fr0057.png +ai_001_001/rgb_cam_00_fr0058.png +ai_001_001/rgb_cam_00_fr0059.png +ai_001_001/rgb_cam_00_fr0060.png +ai_001_001/rgb_cam_00_fr0062.png +ai_001_001/rgb_cam_00_fr0063.png +ai_001_001/rgb_cam_00_fr0064.png +ai_001_001/rgb_cam_00_fr0065.png +ai_001_001/rgb_cam_00_fr0066.png +ai_001_001/rgb_cam_00_fr0067.png +ai_001_001/rgb_cam_00_fr0068.png +ai_001_001/rgb_cam_00_fr0069.png +ai_001_001/rgb_cam_00_fr0070.png +ai_001_001/rgb_cam_00_fr0071.png +ai_001_001/rgb_cam_00_fr0072.png +ai_001_001/rgb_cam_00_fr0073.png +ai_001_001/rgb_cam_00_fr0074.png +ai_001_001/rgb_cam_00_fr0075.png +ai_001_001/rgb_cam_00_fr0076.png +ai_001_001/rgb_cam_00_fr0077.png +ai_001_001/rgb_cam_00_fr0078.png +ai_001_001/rgb_cam_00_fr0079.png +ai_001_001/rgb_cam_00_fr0080.png +ai_001_001/rgb_cam_00_fr0081.png +ai_001_001/rgb_cam_00_fr0082.png +ai_001_001/rgb_cam_00_fr0083.png +ai_001_001/rgb_cam_00_fr0084.png +ai_001_001/rgb_cam_00_fr0085.png +ai_001_001/rgb_cam_00_fr0086.png +ai_001_001/rgb_cam_00_fr0087.png +ai_001_001/rgb_cam_00_fr0088.png +ai_001_001/rgb_cam_00_fr0089.png +ai_001_001/rgb_cam_00_fr0090.png +ai_001_001/rgb_cam_00_fr0091.png +ai_001_001/rgb_cam_00_fr0092.png +ai_001_001/rgb_cam_00_fr0093.png +ai_001_001/rgb_cam_00_fr0094.png +ai_001_001/rgb_cam_00_fr0095.png +ai_001_001/rgb_cam_00_fr0096.png +ai_001_001/rgb_cam_00_fr0097.png +ai_001_001/rgb_cam_00_fr0098.png +ai_001_001/rgb_cam_00_fr0099.png diff --git a/infer/dataset_normal/ibims/__init__.py b/infer/dataset_normal/ibims/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..447602eccdc43136ecb12137ad650c68fa8173c9 --- /dev/null +++ b/infer/dataset_normal/ibims/__init__.py @@ -0,0 +1,47 @@ +""" Get samples from iBims-1 (https://paperswithcode.com/dataset/ibims-1) + NOTE: We computed the GT surface normals by doing discontinuity-aware plane fitting +""" +import os +import cv2 +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" + +from infer.dataset_normal import Sample + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "ibims/corridor_01_img.png" + scene_name = sample_path.split('/')[0] + img_name, img_ext = sample_path.split('/')[1].split('_img') + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'ibims') + img_path = '%s/%s' % (dataset_path, sample_path) + normal_path = img_path.replace('_img'+img_ext, '_normal.exr') + intrins_path = img_path.replace('_img'+img_ext, '_intrins.npy') + assert os.path.exists(img_path) + assert os.path.exists(normal_path) + assert os.path.exists(intrins_path) + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + + # read normal (H, W, 3) + normal = cv2.cvtColor(cv2.imread(normal_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + normal_mask = np.linalg.norm(normal, axis=2, keepdims=True) > 0.5 + + # read intrins (3, 3) + intrins = np.load(intrins_path) + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='ibims', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample \ No newline at end of file diff --git a/infer/dataset_normal/ibims/split/ibims.txt b/infer/dataset_normal/ibims/split/ibims.txt new file mode 100644 index 0000000000000000000000000000000000000000..9cca93a5731b659bdcdf1106a7291631c9d74a95 --- /dev/null +++ b/infer/dataset_normal/ibims/split/ibims.txt @@ -0,0 +1,100 @@ +ibims/corridor_01_img.png +ibims/corridor_02_img.png +ibims/corridor_03_img.png +ibims/corridor_04_img.png +ibims/corridor_05_img.png +ibims/corridor_06_img.png +ibims/corridor_07_img.png +ibims/corridor_08_img.png +ibims/corridor_09_img.png +ibims/corridor_10_img.png +ibims/factory_01_img.png +ibims/factory_02_img.png +ibims/factory_03_img.png +ibims/factory_04_img.png +ibims/factory_05_img.png +ibims/factory_06_img.png +ibims/factory_07_img.png +ibims/factory_08_img.png +ibims/kitchen_01_img.png +ibims/kitchen_02_img.png +ibims/kitchen_03_img.png +ibims/kitchen_04_img.png +ibims/kitchen_05_img.png +ibims/kitchen_06_img.png +ibims/kitchen_07_img.png +ibims/kitchen_08_img.png +ibims/lab_01_img.png +ibims/lab_02_img.png +ibims/lab_03_img.png +ibims/lab_04_img.png +ibims/lab_05_img.png +ibims/lab_06_img.png +ibims/lab_07_img.png +ibims/lab_08_img.png +ibims/lab_09_img.png +ibims/lab_10_img.png +ibims/lab_11_img.png +ibims/lectureroom_01_img.png +ibims/lectureroom_02_img.png +ibims/lectureroom_03_img.png +ibims/lectureroom_04_img.png +ibims/lectureroom_05_img.png +ibims/lectureroom_06_img.png +ibims/lectureroom_07_img.png +ibims/lectureroom_08_img.png +ibims/lectureroom_09_img.png +ibims/lectureroom_10_img.png +ibims/livingroom_01_img.png +ibims/livingroom_02_img.png +ibims/livingroom_03_img.png +ibims/livingroom_04_img.png +ibims/livingroom_05_img.png +ibims/livingroom_06_img.png +ibims/livingroom_07_img.png +ibims/livingroom_08_img.png +ibims/livingroom_09_img.png +ibims/livingroom_10_img.png +ibims/livingroom_11_img.png +ibims/livingroom_12_img.png +ibims/livingroom_13_img.png +ibims/livingroom_14_img.png +ibims/livingroom_15_img.png +ibims/meetingroom_01_img.png +ibims/meetingroom_02_img.png +ibims/meetingroom_03_img.png +ibims/meetingroom_04_img.png +ibims/meetingroom_05_img.png +ibims/meetingroom_06_img.png +ibims/meetingroom_07_img.png +ibims/meetingroom_08_img.png +ibims/office_01_img.png +ibims/office_02_img.png +ibims/office_03_img.png +ibims/office_04_img.png +ibims/office_05_img.png +ibims/office_06_img.png +ibims/office_07_img.png +ibims/office_08_img.png +ibims/restaurant_01_img.png +ibims/restaurant_02_img.png +ibims/restaurant_03_img.png +ibims/restaurant_04_img.png +ibims/restaurant_05_img.png +ibims/restaurant_06_img.png +ibims/restaurant_07_img.png +ibims/restaurant_08_img.png +ibims/restaurant_09_img.png +ibims/restaurant_10_img.png +ibims/restaurant_11_img.png +ibims/restaurant_12_img.png +ibims/restroom_01_img.png +ibims/restroom_02_img.png +ibims/storageroom_01_img.png +ibims/storageroom_02_img.png +ibims/storageroom_03_img.png +ibims/storageroom_04_img.png +ibims/storageroom_05_img.png +ibims/storageroom_06_img.png +ibims/storageroom_07_img.png +ibims/storageroom_08_img.png \ No newline at end of file diff --git a/infer/dataset_normal/normal_dataloader.py b/infer/dataset_normal/normal_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..535a76e0370864d43aa3e64c7f0f0a933d659473 --- /dev/null +++ b/infer/dataset_normal/normal_dataloader.py @@ -0,0 +1,83 @@ +import os +import random + +import torch +from torch.utils.data import Dataset +from torch.utils.data import DataLoader +from torchvision import transforms + +from . import aug_basic + +import logging +logger = logging.getLogger('root') + + +def get_transform(dataset_name='hypersim', mode='test'): + assert mode in ['test'] + logger.info('Defining %s transform for %s dataset' % (mode, dataset_name)) + tf_list = [ + aug_basic.ToTensor(), + ] + tf_list += [ + # 选项1：使用标准归一化 (如果启用，需要相应调整可视化代码) + # aug_basic.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + # 选项2：不使用归一化 (当前方案，配合修改后的unnormalize函数) + aug_basic.ToDict(), + ] + logger.info('Defining %s transform for %s dataset ... DONE' % (mode, dataset_name)) + return transforms.Compose(tf_list) + + +class NormalDataset(Dataset): + def __init__(self, base_data_dir, dataset_split_path, dataset_name='nyuv2', split='test', mode='test', epoch=0): + self.split = split + self.mode = mode + self.base_data_dir = base_data_dir + assert mode in ['test'] + + # data split + split_path = os.path.join(dataset_split_path, dataset_name, 'split', split+'.txt') # dataset_split_path: eval/dataset_normal/ + assert os.path.exists(split_path) + with open(split_path, 'r') as f: + self.filenames = [i.strip() for i in f.readlines()] + self.split_path = split_path + + # get_sample function + if dataset_name == 'nyuv2': + from infer.dataset_normal.nyuv2 import get_sample + elif dataset_name == 'scannet': + from infer.dataset_normal.scannet import get_sample + elif dataset_name == 'ibims': + from infer.dataset_normal.ibims import get_sample + elif dataset_name == 'sintel': + from infer.dataset_normal.sintel import get_sample + elif dataset_name == 'oasis': + from infer.dataset_normal.oasis import get_sample + elif dataset_name == 'hypersim': + from infer.dataset_normal.hypersim import get_sample + else: + raise NotImplementedError(f"Unsupported normal dataset: {dataset_name}") + self.get_sample = get_sample + + # data preprocessing/augmentation + self.transform = get_transform(dataset_name=dataset_name, mode=mode) + + def __len__(self): + return len(self.filenames) + + def __getitem__(self, index): + info = {} + + sample = self.transform(self.get_sample( + base_data_dir = self.base_data_dir, + sample_path=self.filenames[index], + info=info) + ) + + return sample + +class TestLoader(object): + def __init__(self, base_data_dir, dataset_split_path, dataset_name_test, test_split): + self.test_samples = NormalDataset(base_data_dir, dataset_split_path, dataset_name=dataset_name_test, + split=test_split, mode='test', epoch=0) + self.data = DataLoader(self.test_samples, 1, shuffle=False, num_workers=4, pin_memory=True) diff --git a/infer/dataset_normal/nyuv2/__init__.py b/infer/dataset_normal/nyuv2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..19ed835b4b5f7fc0cfbfbd3d92222645d027a746 --- /dev/null +++ b/infer/dataset_normal/nyuv2/__init__.py @@ -0,0 +1,66 @@ +""" Get samples from NYUv2 (https://cs.nyu.edu/~fergus/datasets/nyu_depth_v2.html) + NOTE: GT surface normals are from GeoNet (CVPR 2018) - https://github.com/xjqi/GeoNet +""" +import os +import cv2 +import numpy as np + +from infer.dataset_normal import Sample + + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "test/000000_img.png" + scene_name = sample_path.split('/')[0] + img_name, img_ext = sample_path.split('/')[1].split('_img') + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'nyuv2') + img_path = '%s/%s' % (dataset_path, sample_path) + normal_png_path = img_path.replace('_img'+img_ext, '_normal.png') + normal_npy_path = img_path.replace('_img'+img_ext, '_normal.npy') + intrins_path = img_path.replace('_img'+img_ext, '_intrins.npy') + assert os.path.exists(img_path) + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + #保存图像 + # cv2.imwrite(os.path.join(base_data_dir, img_name+'_img.png'), img*255) + + # read normal (H, W, 3) + if os.path.exists(normal_png_path): + normal = cv2.cvtColor(cv2.imread(normal_png_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + normal_mask = np.sum(normal, axis=2, keepdims=True) > 0 + normal = (normal.astype(np.float32) / 255.0) * 2.0 - 1.0 + elif os.path.exists(normal_npy_path): + normal = np.load(normal_npy_path).astype(np.float32) + assert normal.ndim == 3 and normal.shape[2] == 3, f"Unexpected normal shape: {normal.shape}" + # GeoNet npy normals use opposite x-axis convention for this evaluation codepath. + normal[:, :, 0] *= -1.0 + normal_mask = np.linalg.norm(normal, axis=2, keepdims=True) > 1e-6 + else: + raise FileNotFoundError(f"Missing NYUv2 normal file: {normal_png_path} or {normal_npy_path}") + + # read intrins (3, 3) + if os.path.exists(intrins_path): + intrins = np.load(intrins_path) + else: + # Fallback to NYUv2 default intrinsics used by many benchmarks. + intrins = np.array([ + [518.8579, 0.0, 325.5824], + [0.0, 519.4696, 253.7362], + [0.0, 0.0, 1.0], + ], dtype=np.float32) + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='nyuv2', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample diff --git a/infer/dataset_normal/nyuv2/split/test.txt b/infer/dataset_normal/nyuv2/split/test.txt new file mode 100644 index 0000000000000000000000000000000000000000..1ee98378c85a069a8fae059169a1118ad50712a3 --- /dev/null +++ b/infer/dataset_normal/nyuv2/split/test.txt @@ -0,0 +1,654 @@ +test/000000_img.png +test/000001_img.png +test/000008_img.png +test/000013_img.png +test/000014_img.png +test/000015_img.png +test/000016_img.png +test/000017_img.png +test/000020_img.png +test/000027_img.png +test/000028_img.png +test/000029_img.png +test/000030_img.png +test/000031_img.png +test/000032_img.png +test/000033_img.png +test/000034_img.png +test/000035_img.png +test/000036_img.png +test/000037_img.png +test/000038_img.png +test/000039_img.png +test/000040_img.png +test/000041_img.png +test/000042_img.png +test/000045_img.png +test/000046_img.png +test/000055_img.png +test/000056_img.png +test/000058_img.png +test/000059_img.png +test/000060_img.png +test/000061_img.png +test/000062_img.png +test/000075_img.png +test/000076_img.png +test/000077_img.png +test/000078_img.png +test/000083_img.png +test/000084_img.png +test/000085_img.png +test/000086_img.png +test/000087_img.png +test/000088_img.png +test/000089_img.png +test/000090_img.png +test/000116_img.png +test/000117_img.png +test/000118_img.png +test/000124_img.png +test/000125_img.png +test/000126_img.png +test/000127_img.png +test/000128_img.png +test/000130_img.png +test/000131_img.png +test/000132_img.png +test/000133_img.png +test/000136_img.png +test/000152_img.png +test/000153_img.png +test/000154_img.png +test/000166_img.png +test/000167_img.png +test/000168_img.png +test/000170_img.png +test/000171_img.png +test/000172_img.png +test/000173_img.png +test/000174_img.png +test/000175_img.png +test/000179_img.png +test/000180_img.png +test/000181_img.png +test/000182_img.png +test/000183_img.png +test/000184_img.png +test/000185_img.png +test/000186_img.png +test/000187_img.png +test/000188_img.png +test/000189_img.png +test/000190_img.png +test/000191_img.png +test/000192_img.png +test/000193_img.png +test/000194_img.png +test/000195_img.png +test/000196_img.png +test/000197_img.png +test/000198_img.png +test/000199_img.png +test/000200_img.png +test/000201_img.png +test/000206_img.png +test/000207_img.png +test/000208_img.png +test/000209_img.png +test/000210_img.png +test/000211_img.png +test/000219_img.png +test/000220_img.png +test/000221_img.png +test/000249_img.png +test/000263_img.png +test/000270_img.png +test/000271_img.png +test/000272_img.png +test/000278_img.png +test/000279_img.png +test/000280_img.png +test/000281_img.png +test/000282_img.png +test/000283_img.png +test/000284_img.png +test/000295_img.png +test/000296_img.png +test/000297_img.png +test/000298_img.png +test/000299_img.png +test/000300_img.png +test/000301_img.png +test/000309_img.png +test/000310_img.png +test/000311_img.png +test/000314_img.png +test/000315_img.png +test/000316_img.png +test/000324_img.png +test/000325_img.png +test/000326_img.png +test/000327_img.png +test/000328_img.png +test/000329_img.png +test/000330_img.png +test/000331_img.png +test/000332_img.png +test/000333_img.png +test/000334_img.png +test/000350_img.png +test/000351_img.png +test/000354_img.png +test/000355_img.png +test/000356_img.png +test/000357_img.png +test/000358_img.png +test/000359_img.png +test/000360_img.png +test/000361_img.png +test/000362_img.png +test/000363_img.png +test/000383_img.png +test/000384_img.png +test/000385_img.png +test/000386_img.png +test/000387_img.png +test/000388_img.png +test/000389_img.png +test/000394_img.png +test/000395_img.png +test/000396_img.png +test/000410_img.png +test/000411_img.png +test/000412_img.png +test/000413_img.png +test/000429_img.png +test/000430_img.png +test/000431_img.png +test/000432_img.png +test/000433_img.png +test/000434_img.png +test/000440_img.png +test/000441_img.png +test/000442_img.png +test/000443_img.png +test/000444_img.png +test/000445_img.png +test/000446_img.png +test/000447_img.png +test/000461_img.png +test/000462_img.png +test/000463_img.png +test/000464_img.png +test/000465_img.png +test/000468_img.png +test/000469_img.png +test/000470_img.png +test/000471_img.png +test/000472_img.png +test/000473_img.png +test/000474_img.png +test/000475_img.png +test/000476_img.png +test/000507_img.png +test/000508_img.png +test/000509_img.png +test/000510_img.png +test/000511_img.png +test/000512_img.png +test/000514_img.png +test/000515_img.png +test/000516_img.png +test/000517_img.png +test/000518_img.png +test/000519_img.png +test/000520_img.png +test/000521_img.png +test/000522_img.png +test/000523_img.png +test/000524_img.png +test/000525_img.png +test/000530_img.png +test/000531_img.png +test/000532_img.png +test/000536_img.png +test/000537_img.png +test/000538_img.png +test/000548_img.png +test/000549_img.png +test/000550_img.png +test/000554_img.png +test/000555_img.png +test/000556_img.png +test/000557_img.png +test/000558_img.png +test/000559_img.png +test/000560_img.png +test/000561_img.png +test/000562_img.png +test/000563_img.png +test/000564_img.png +test/000565_img.png +test/000566_img.png +test/000567_img.png +test/000568_img.png +test/000569_img.png +test/000570_img.png +test/000578_img.png +test/000579_img.png +test/000580_img.png +test/000581_img.png +test/000582_img.png +test/000590_img.png +test/000591_img.png +test/000592_img.png +test/000593_img.png +test/000602_img.png +test/000603_img.png +test/000604_img.png +test/000605_img.png +test/000606_img.png +test/000611_img.png +test/000612_img.png +test/000616_img.png +test/000617_img.png +test/000618_img.png +test/000619_img.png +test/000620_img.png +test/000632_img.png +test/000633_img.png +test/000634_img.png +test/000635_img.png +test/000636_img.png +test/000637_img.png +test/000643_img.png +test/000644_img.png +test/000649_img.png +test/000650_img.png +test/000655_img.png +test/000656_img.png +test/000657_img.png +test/000662_img.png +test/000663_img.png +test/000667_img.png +test/000668_img.png +test/000669_img.png +test/000670_img.png +test/000671_img.png +test/000672_img.png +test/000675_img.png +test/000676_img.png +test/000677_img.png +test/000678_img.png +test/000679_img.png +test/000680_img.png +test/000685_img.png +test/000686_img.png +test/000687_img.png +test/000688_img.png +test/000689_img.png +test/000692_img.png +test/000693_img.png +test/000696_img.png +test/000697_img.png +test/000698_img.png +test/000705_img.png +test/000706_img.png +test/000707_img.png +test/000708_img.png +test/000709_img.png +test/000710_img.png +test/000711_img.png +test/000712_img.png +test/000716_img.png +test/000717_img.png +test/000723_img.png +test/000724_img.png +test/000725_img.png +test/000726_img.png +test/000727_img.png +test/000730_img.png +test/000731_img.png +test/000732_img.png +test/000733_img.png +test/000742_img.png +test/000743_img.png +test/000758_img.png +test/000759_img.png +test/000760_img.png +test/000761_img.png +test/000762_img.png +test/000763_img.png +test/000764_img.png +test/000765_img.png +test/000766_img.png +test/000767_img.png +test/000768_img.png +test/000769_img.png +test/000770_img.png +test/000771_img.png +test/000772_img.png +test/000773_img.png +test/000774_img.png +test/000775_img.png +test/000776_img.png +test/000777_img.png +test/000778_img.png +test/000779_img.png +test/000780_img.png +test/000781_img.png +test/000782_img.png +test/000783_img.png +test/000784_img.png +test/000785_img.png +test/000786_img.png +test/000799_img.png +test/000800_img.png +test/000801_img.png +test/000802_img.png +test/000803_img.png +test/000809_img.png +test/000810_img.png +test/000811_img.png +test/000812_img.png +test/000813_img.png +test/000820_img.png +test/000821_img.png +test/000822_img.png +test/000832_img.png +test/000833_img.png +test/000834_img.png +test/000835_img.png +test/000836_img.png +test/000837_img.png +test/000838_img.png +test/000839_img.png +test/000840_img.png +test/000841_img.png +test/000842_img.png +test/000843_img.png +test/000844_img.png +test/000845_img.png +test/000849_img.png +test/000850_img.png +test/000851_img.png +test/000856_img.png +test/000857_img.png +test/000858_img.png +test/000859_img.png +test/000860_img.png +test/000861_img.png +test/000868_img.png +test/000869_img.png +test/000870_img.png +test/000905_img.png +test/000906_img.png +test/000907_img.png +test/000916_img.png +test/000917_img.png +test/000918_img.png +test/000925_img.png +test/000926_img.png +test/000927_img.png +test/000931_img.png +test/000932_img.png +test/000933_img.png +test/000934_img.png +test/000944_img.png +test/000945_img.png +test/000946_img.png +test/000958_img.png +test/000959_img.png +test/000960_img.png +test/000961_img.png +test/000964_img.png +test/000965_img.png +test/000966_img.png +test/000969_img.png +test/000970_img.png +test/000971_img.png +test/000972_img.png +test/000973_img.png +test/000974_img.png +test/000975_img.png +test/000976_img.png +test/000990_img.png +test/000991_img.png +test/000992_img.png +test/000993_img.png +test/000994_img.png +test/001000_img.png +test/001001_img.png +test/001002_img.png +test/001003_img.png +test/001009_img.png +test/001010_img.png +test/001011_img.png +test/001020_img.png +test/001021_img.png +test/001022_img.png +test/001031_img.png +test/001032_img.png +test/001033_img.png +test/001037_img.png +test/001038_img.png +test/001047_img.png +test/001048_img.png +test/001051_img.png +test/001052_img.png +test/001056_img.png +test/001057_img.png +test/001074_img.png +test/001075_img.png +test/001076_img.png +test/001077_img.png +test/001078_img.png +test/001079_img.png +test/001080_img.png +test/001081_img.png +test/001082_img.png +test/001083_img.png +test/001087_img.png +test/001088_img.png +test/001089_img.png +test/001090_img.png +test/001091_img.png +test/001092_img.png +test/001093_img.png +test/001094_img.png +test/001095_img.png +test/001097_img.png +test/001098_img.png +test/001099_img.png +test/001100_img.png +test/001101_img.png +test/001102_img.png +test/001103_img.png +test/001105_img.png +test/001106_img.png +test/001107_img.png +test/001108_img.png +test/001116_img.png +test/001117_img.png +test/001118_img.png +test/001122_img.png +test/001123_img.png +test/001124_img.png +test/001125_img.png +test/001126_img.png +test/001127_img.png +test/001128_img.png +test/001129_img.png +test/001130_img.png +test/001134_img.png +test/001135_img.png +test/001143_img.png +test/001144_img.png +test/001145_img.png +test/001146_img.png +test/001147_img.png +test/001148_img.png +test/001149_img.png +test/001150_img.png +test/001151_img.png +test/001152_img.png +test/001153_img.png +test/001154_img.png +test/001155_img.png +test/001156_img.png +test/001157_img.png +test/001161_img.png +test/001162_img.png +test/001163_img.png +test/001164_img.png +test/001165_img.png +test/001166_img.png +test/001169_img.png +test/001170_img.png +test/001173_img.png +test/001174_img.png +test/001175_img.png +test/001178_img.png +test/001179_img.png +test/001180_img.png +test/001181_img.png +test/001182_img.png +test/001183_img.png +test/001191_img.png +test/001192_img.png +test/001193_img.png +test/001194_img.png +test/001195_img.png +test/001200_img.png +test/001201_img.png +test/001202_img.png +test/001203_img.png +test/001204_img.png +test/001205_img.png +test/001206_img.png +test/001207_img.png +test/001208_img.png +test/001209_img.png +test/001210_img.png +test/001211_img.png +test/001215_img.png +test/001216_img.png +test/001217_img.png +test/001218_img.png +test/001219_img.png +test/001225_img.png +test/001226_img.png +test/001227_img.png +test/001228_img.png +test/001229_img.png +test/001232_img.png +test/001233_img.png +test/001234_img.png +test/001246_img.png +test/001247_img.png +test/001248_img.png +test/001249_img.png +test/001253_img.png +test/001254_img.png +test/001255_img.png +test/001256_img.png +test/001257_img.png +test/001258_img.png +test/001259_img.png +test/001260_img.png +test/001261_img.png +test/001262_img.png +test/001263_img.png +test/001264_img.png +test/001274_img.png +test/001275_img.png +test/001276_img.png +test/001277_img.png +test/001278_img.png +test/001279_img.png +test/001284_img.png +test/001285_img.png +test/001286_img.png +test/001287_img.png +test/001288_img.png +test/001289_img.png +test/001290_img.png +test/001291_img.png +test/001292_img.png +test/001293_img.png +test/001294_img.png +test/001296_img.png +test/001297_img.png +test/001298_img.png +test/001301_img.png +test/001302_img.png +test/001303_img.png +test/001304_img.png +test/001305_img.png +test/001306_img.png +test/001307_img.png +test/001313_img.png +test/001314_img.png +test/001328_img.png +test/001329_img.png +test/001330_img.png +test/001331_img.png +test/001334_img.png +test/001335_img.png +test/001336_img.png +test/001337_img.png +test/001338_img.png +test/001339_img.png +test/001346_img.png +test/001347_img.png +test/001348_img.png +test/001352_img.png +test/001353_img.png +test/001354_img.png +test/001355_img.png +test/001363_img.png +test/001364_img.png +test/001367_img.png +test/001368_img.png +test/001383_img.png +test/001384_img.png +test/001385_img.png +test/001386_img.png +test/001387_img.png +test/001388_img.png +test/001389_img.png +test/001390_img.png +test/001393_img.png +test/001394_img.png +test/001395_img.png +test/001396_img.png +test/001397_img.png +test/001398_img.png +test/001399_img.png +test/001400_img.png +test/001406_img.png +test/001407_img.png +test/001408_img.png +test/001409_img.png +test/001410_img.png +test/001411_img.png +test/001412_img.png +test/001413_img.png +test/001420_img.png +test/001421_img.png +test/001422_img.png +test/001423_img.png +test/001429_img.png +test/001430_img.png +test/001431_img.png +test/001432_img.png +test/001440_img.png +test/001441_img.png +test/001442_img.png +test/001443_img.png +test/001444_img.png +test/001445_img.png +test/001446_img.png +test/001447_img.png +test/001448_img.png \ No newline at end of file diff --git a/infer/dataset_normal/nyuv2/split/train.txt b/infer/dataset_normal/nyuv2/split/train.txt new file mode 100644 index 0000000000000000000000000000000000000000..f03309626327e00831f870c666871f6fe2c9e890 --- /dev/null +++ b/infer/dataset_normal/nyuv2/split/train.txt @@ -0,0 +1,795 @@ +train/000002_img.png +train/000003_img.png +train/000004_img.png +train/000005_img.png +train/000006_img.png +train/000007_img.png +train/000009_img.png +train/000010_img.png +train/000011_img.png +train/000012_img.png +train/000018_img.png +train/000019_img.png +train/000021_img.png +train/000022_img.png +train/000023_img.png +train/000024_img.png +train/000025_img.png +train/000026_img.png +train/000043_img.png +train/000044_img.png +train/000047_img.png +train/000048_img.png +train/000049_img.png +train/000050_img.png +train/000051_img.png +train/000052_img.png +train/000053_img.png +train/000054_img.png +train/000057_img.png +train/000063_img.png +train/000064_img.png +train/000065_img.png +train/000066_img.png +train/000067_img.png +train/000068_img.png +train/000069_img.png +train/000070_img.png +train/000071_img.png +train/000072_img.png +train/000073_img.png +train/000074_img.png +train/000079_img.png +train/000080_img.png +train/000081_img.png +train/000082_img.png +train/000091_img.png +train/000092_img.png +train/000093_img.png +train/000094_img.png +train/000095_img.png +train/000096_img.png +train/000097_img.png +train/000098_img.png +train/000099_img.png +train/000100_img.png +train/000101_img.png +train/000102_img.png +train/000103_img.png +train/000104_img.png +train/000105_img.png +train/000106_img.png +train/000107_img.png +train/000108_img.png +train/000109_img.png +train/000110_img.png +train/000111_img.png +train/000112_img.png +train/000113_img.png +train/000114_img.png +train/000115_img.png +train/000119_img.png +train/000120_img.png +train/000121_img.png +train/000122_img.png +train/000123_img.png +train/000129_img.png +train/000134_img.png +train/000135_img.png +train/000137_img.png +train/000138_img.png +train/000139_img.png +train/000140_img.png +train/000141_img.png +train/000142_img.png +train/000143_img.png +train/000144_img.png +train/000145_img.png +train/000146_img.png +train/000147_img.png +train/000148_img.png +train/000149_img.png +train/000150_img.png +train/000151_img.png +train/000155_img.png +train/000156_img.png +train/000157_img.png +train/000158_img.png +train/000159_img.png +train/000160_img.png +train/000161_img.png +train/000162_img.png +train/000163_img.png +train/000164_img.png +train/000165_img.png +train/000169_img.png +train/000176_img.png +train/000177_img.png +train/000178_img.png +train/000202_img.png +train/000203_img.png +train/000204_img.png +train/000205_img.png +train/000212_img.png +train/000213_img.png +train/000214_img.png +train/000215_img.png +train/000216_img.png +train/000217_img.png +train/000218_img.png +train/000222_img.png +train/000223_img.png +train/000224_img.png +train/000225_img.png +train/000226_img.png +train/000227_img.png +train/000228_img.png +train/000229_img.png +train/000230_img.png +train/000231_img.png +train/000232_img.png +train/000233_img.png +train/000234_img.png +train/000235_img.png +train/000236_img.png +train/000237_img.png +train/000238_img.png +train/000239_img.png +train/000240_img.png +train/000241_img.png +train/000242_img.png +train/000243_img.png +train/000244_img.png +train/000245_img.png +train/000246_img.png +train/000247_img.png +train/000248_img.png +train/000250_img.png +train/000251_img.png +train/000252_img.png +train/000253_img.png +train/000254_img.png +train/000255_img.png +train/000256_img.png +train/000257_img.png +train/000258_img.png +train/000259_img.png +train/000260_img.png +train/000261_img.png +train/000262_img.png +train/000264_img.png +train/000265_img.png +train/000266_img.png +train/000267_img.png +train/000268_img.png +train/000269_img.png +train/000273_img.png +train/000274_img.png +train/000275_img.png +train/000276_img.png +train/000277_img.png +train/000285_img.png +train/000286_img.png +train/000287_img.png +train/000288_img.png +train/000289_img.png +train/000290_img.png +train/000291_img.png +train/000292_img.png +train/000293_img.png +train/000294_img.png +train/000302_img.png +train/000303_img.png +train/000304_img.png +train/000305_img.png +train/000306_img.png +train/000307_img.png +train/000308_img.png +train/000312_img.png +train/000313_img.png +train/000317_img.png +train/000318_img.png +train/000319_img.png +train/000320_img.png +train/000321_img.png +train/000322_img.png +train/000323_img.png +train/000335_img.png +train/000336_img.png +train/000337_img.png +train/000338_img.png +train/000339_img.png +train/000340_img.png +train/000341_img.png +train/000342_img.png +train/000343_img.png +train/000344_img.png +train/000345_img.png +train/000346_img.png +train/000347_img.png +train/000348_img.png +train/000349_img.png +train/000352_img.png +train/000353_img.png +train/000364_img.png +train/000365_img.png +train/000366_img.png +train/000367_img.png +train/000368_img.png +train/000369_img.png +train/000370_img.png +train/000371_img.png +train/000372_img.png +train/000373_img.png +train/000374_img.png +train/000375_img.png +train/000376_img.png +train/000377_img.png +train/000378_img.png +train/000379_img.png +train/000380_img.png +train/000381_img.png +train/000382_img.png +train/000390_img.png +train/000391_img.png +train/000392_img.png +train/000393_img.png +train/000397_img.png +train/000398_img.png +train/000399_img.png +train/000400_img.png +train/000401_img.png +train/000402_img.png +train/000403_img.png +train/000404_img.png +train/000405_img.png +train/000406_img.png +train/000407_img.png +train/000408_img.png +train/000409_img.png +train/000414_img.png +train/000415_img.png +train/000416_img.png +train/000417_img.png +train/000418_img.png +train/000419_img.png +train/000420_img.png +train/000421_img.png +train/000422_img.png +train/000423_img.png +train/000424_img.png +train/000425_img.png +train/000426_img.png +train/000427_img.png +train/000428_img.png +train/000435_img.png +train/000436_img.png +train/000437_img.png +train/000438_img.png +train/000439_img.png +train/000448_img.png +train/000449_img.png +train/000450_img.png +train/000451_img.png +train/000452_img.png +train/000453_img.png +train/000454_img.png +train/000455_img.png +train/000456_img.png +train/000457_img.png +train/000458_img.png +train/000459_img.png +train/000460_img.png +train/000466_img.png +train/000467_img.png +train/000477_img.png +train/000478_img.png +train/000479_img.png +train/000480_img.png +train/000481_img.png +train/000482_img.png +train/000483_img.png +train/000484_img.png +train/000485_img.png +train/000486_img.png +train/000487_img.png +train/000488_img.png +train/000489_img.png +train/000490_img.png +train/000491_img.png +train/000492_img.png +train/000493_img.png +train/000494_img.png +train/000495_img.png +train/000496_img.png +train/000497_img.png +train/000498_img.png +train/000499_img.png +train/000500_img.png +train/000501_img.png +train/000502_img.png +train/000503_img.png +train/000504_img.png +train/000505_img.png +train/000506_img.png +train/000513_img.png +train/000526_img.png +train/000527_img.png +train/000528_img.png +train/000529_img.png +train/000533_img.png +train/000534_img.png +train/000535_img.png +train/000539_img.png +train/000540_img.png +train/000541_img.png +train/000542_img.png +train/000543_img.png +train/000544_img.png +train/000545_img.png +train/000546_img.png +train/000547_img.png +train/000551_img.png +train/000552_img.png +train/000553_img.png +train/000571_img.png +train/000572_img.png +train/000573_img.png +train/000574_img.png +train/000575_img.png +train/000576_img.png +train/000577_img.png +train/000583_img.png +train/000584_img.png +train/000585_img.png +train/000586_img.png +train/000587_img.png +train/000588_img.png +train/000589_img.png +train/000594_img.png +train/000595_img.png +train/000596_img.png +train/000597_img.png +train/000598_img.png +train/000599_img.png +train/000600_img.png +train/000601_img.png +train/000607_img.png +train/000608_img.png +train/000609_img.png +train/000610_img.png +train/000613_img.png +train/000614_img.png +train/000615_img.png +train/000621_img.png +train/000622_img.png +train/000623_img.png +train/000624_img.png +train/000625_img.png +train/000626_img.png +train/000627_img.png +train/000628_img.png +train/000629_img.png +train/000630_img.png +train/000631_img.png +train/000638_img.png +train/000639_img.png +train/000640_img.png +train/000641_img.png +train/000642_img.png +train/000645_img.png +train/000646_img.png +train/000647_img.png +train/000648_img.png +train/000651_img.png +train/000652_img.png +train/000653_img.png +train/000654_img.png +train/000658_img.png +train/000659_img.png +train/000660_img.png +train/000661_img.png +train/000664_img.png +train/000665_img.png +train/000666_img.png +train/000673_img.png +train/000674_img.png +train/000681_img.png +train/000682_img.png +train/000683_img.png +train/000684_img.png +train/000690_img.png +train/000691_img.png +train/000694_img.png +train/000695_img.png +train/000699_img.png +train/000700_img.png +train/000701_img.png +train/000702_img.png +train/000703_img.png +train/000704_img.png +train/000713_img.png +train/000714_img.png +train/000715_img.png +train/000718_img.png +train/000719_img.png +train/000720_img.png +train/000721_img.png +train/000722_img.png +train/000728_img.png +train/000729_img.png +train/000734_img.png +train/000735_img.png +train/000736_img.png +train/000737_img.png +train/000738_img.png +train/000739_img.png +train/000740_img.png +train/000741_img.png +train/000744_img.png +train/000745_img.png +train/000746_img.png +train/000747_img.png +train/000748_img.png +train/000749_img.png +train/000750_img.png +train/000751_img.png +train/000752_img.png +train/000753_img.png +train/000754_img.png +train/000755_img.png +train/000756_img.png +train/000757_img.png +train/000787_img.png +train/000788_img.png +train/000789_img.png +train/000790_img.png +train/000791_img.png +train/000792_img.png +train/000793_img.png +train/000794_img.png +train/000795_img.png +train/000796_img.png +train/000797_img.png +train/000798_img.png +train/000804_img.png +train/000805_img.png +train/000806_img.png +train/000807_img.png +train/000808_img.png +train/000814_img.png +train/000815_img.png +train/000816_img.png +train/000817_img.png +train/000818_img.png +train/000819_img.png +train/000823_img.png +train/000824_img.png +train/000825_img.png +train/000826_img.png +train/000827_img.png +train/000828_img.png +train/000829_img.png +train/000830_img.png +train/000831_img.png +train/000846_img.png +train/000847_img.png +train/000848_img.png +train/000852_img.png +train/000853_img.png +train/000854_img.png +train/000855_img.png +train/000862_img.png +train/000863_img.png +train/000864_img.png +train/000865_img.png +train/000866_img.png +train/000867_img.png +train/000871_img.png +train/000872_img.png +train/000873_img.png +train/000874_img.png +train/000875_img.png +train/000876_img.png +train/000877_img.png +train/000878_img.png +train/000879_img.png +train/000880_img.png +train/000881_img.png +train/000882_img.png +train/000883_img.png +train/000884_img.png +train/000885_img.png +train/000886_img.png +train/000887_img.png +train/000888_img.png +train/000889_img.png +train/000890_img.png +train/000891_img.png +train/000892_img.png +train/000893_img.png +train/000894_img.png +train/000895_img.png +train/000896_img.png +train/000897_img.png +train/000898_img.png +train/000899_img.png +train/000900_img.png +train/000901_img.png +train/000902_img.png +train/000903_img.png +train/000904_img.png +train/000908_img.png +train/000909_img.png +train/000910_img.png +train/000911_img.png +train/000912_img.png +train/000913_img.png +train/000914_img.png +train/000915_img.png +train/000919_img.png +train/000920_img.png +train/000921_img.png +train/000922_img.png +train/000923_img.png +train/000924_img.png +train/000928_img.png +train/000929_img.png +train/000930_img.png +train/000935_img.png +train/000936_img.png +train/000937_img.png +train/000938_img.png +train/000939_img.png +train/000940_img.png +train/000941_img.png +train/000942_img.png +train/000943_img.png +train/000947_img.png +train/000948_img.png +train/000949_img.png +train/000950_img.png +train/000951_img.png +train/000952_img.png +train/000953_img.png +train/000954_img.png +train/000955_img.png +train/000956_img.png +train/000957_img.png +train/000962_img.png +train/000963_img.png +train/000967_img.png +train/000968_img.png +train/000977_img.png +train/000978_img.png +train/000979_img.png +train/000980_img.png +train/000981_img.png +train/000982_img.png +train/000983_img.png +train/000984_img.png +train/000985_img.png +train/000986_img.png +train/000987_img.png +train/000988_img.png +train/000989_img.png +train/000995_img.png +train/000996_img.png +train/000997_img.png +train/000998_img.png +train/000999_img.png +train/001004_img.png +train/001005_img.png +train/001006_img.png +train/001007_img.png +train/001008_img.png +train/001012_img.png +train/001013_img.png +train/001014_img.png +train/001015_img.png +train/001016_img.png +train/001017_img.png +train/001018_img.png +train/001019_img.png +train/001023_img.png +train/001024_img.png +train/001025_img.png +train/001026_img.png +train/001027_img.png +train/001028_img.png +train/001029_img.png +train/001030_img.png +train/001034_img.png +train/001035_img.png +train/001036_img.png +train/001039_img.png +train/001040_img.png +train/001041_img.png +train/001042_img.png +train/001043_img.png +train/001044_img.png +train/001045_img.png +train/001046_img.png +train/001049_img.png +train/001050_img.png +train/001053_img.png +train/001054_img.png +train/001055_img.png +train/001058_img.png +train/001059_img.png +train/001060_img.png +train/001061_img.png +train/001062_img.png +train/001063_img.png +train/001064_img.png +train/001065_img.png +train/001066_img.png +train/001067_img.png +train/001068_img.png +train/001069_img.png +train/001070_img.png +train/001071_img.png +train/001072_img.png +train/001073_img.png +train/001084_img.png +train/001085_img.png +train/001086_img.png +train/001096_img.png +train/001104_img.png +train/001109_img.png +train/001110_img.png +train/001111_img.png +train/001112_img.png +train/001113_img.png +train/001114_img.png +train/001115_img.png +train/001119_img.png +train/001120_img.png +train/001121_img.png +train/001131_img.png +train/001132_img.png +train/001133_img.png +train/001136_img.png +train/001137_img.png +train/001138_img.png +train/001139_img.png +train/001140_img.png +train/001141_img.png +train/001142_img.png +train/001158_img.png +train/001159_img.png +train/001160_img.png +train/001167_img.png +train/001168_img.png +train/001171_img.png +train/001172_img.png +train/001176_img.png +train/001177_img.png +train/001184_img.png +train/001185_img.png +train/001186_img.png +train/001187_img.png +train/001188_img.png +train/001189_img.png +train/001190_img.png +train/001196_img.png +train/001197_img.png +train/001198_img.png +train/001199_img.png +train/001212_img.png +train/001213_img.png +train/001214_img.png +train/001220_img.png +train/001221_img.png +train/001222_img.png +train/001223_img.png +train/001224_img.png +train/001230_img.png +train/001231_img.png +train/001235_img.png +train/001236_img.png +train/001237_img.png +train/001238_img.png +train/001239_img.png +train/001240_img.png +train/001241_img.png +train/001242_img.png +train/001243_img.png +train/001244_img.png +train/001245_img.png +train/001250_img.png +train/001251_img.png +train/001252_img.png +train/001265_img.png +train/001266_img.png +train/001267_img.png +train/001268_img.png +train/001269_img.png +train/001270_img.png +train/001271_img.png +train/001272_img.png +train/001273_img.png +train/001280_img.png +train/001281_img.png +train/001282_img.png +train/001283_img.png +train/001295_img.png +train/001299_img.png +train/001300_img.png +train/001308_img.png +train/001309_img.png +train/001310_img.png +train/001311_img.png +train/001312_img.png +train/001315_img.png +train/001316_img.png +train/001317_img.png +train/001318_img.png +train/001319_img.png +train/001320_img.png +train/001321_img.png +train/001322_img.png +train/001323_img.png +train/001324_img.png +train/001325_img.png +train/001326_img.png +train/001327_img.png +train/001332_img.png +train/001333_img.png +train/001340_img.png +train/001341_img.png +train/001342_img.png +train/001343_img.png +train/001344_img.png +train/001345_img.png +train/001349_img.png +train/001350_img.png +train/001351_img.png +train/001356_img.png +train/001357_img.png +train/001358_img.png +train/001359_img.png +train/001360_img.png +train/001361_img.png +train/001362_img.png +train/001365_img.png +train/001366_img.png +train/001369_img.png +train/001370_img.png +train/001371_img.png +train/001372_img.png +train/001373_img.png +train/001374_img.png +train/001375_img.png +train/001376_img.png +train/001377_img.png +train/001378_img.png +train/001379_img.png +train/001380_img.png +train/001381_img.png +train/001382_img.png +train/001391_img.png +train/001392_img.png +train/001401_img.png +train/001402_img.png +train/001403_img.png +train/001404_img.png +train/001405_img.png +train/001414_img.png +train/001415_img.png +train/001416_img.png +train/001417_img.png +train/001418_img.png +train/001419_img.png +train/001424_img.png +train/001425_img.png +train/001426_img.png +train/001427_img.png +train/001428_img.png +train/001433_img.png +train/001434_img.png +train/001435_img.png +train/001436_img.png +train/001437_img.png +train/001438_img.png +train/001439_img.png \ No newline at end of file diff --git a/infer/dataset_normal/oasis/__init__.py b/infer/dataset_normal/oasis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b72e3adf835b29935ed2233973a93d3e3f2bbbf8 --- /dev/null +++ b/infer/dataset_normal/oasis/__init__.py @@ -0,0 +1,76 @@ +""" Get samples from OASIS validation set (https://pvl.cs.princeton.edu/OASIS/) +""" +import os +import cv2 +import numpy as np +import pickle + +from infer.dataset_normal import Sample + + +def read_normal(path, h, w): + normal_dict = pickle.load(open(path, 'rb')) + + mask = np.zeros((h,w)) + normal = np.zeros((h,w,3)) + + # Stuff ROI normal into bounding box + min_y = normal_dict['min_y'] + max_y = normal_dict['max_y'] + min_x = normal_dict['min_x'] + max_x = normal_dict['max_x'] + roi_normal = normal_dict['normal'] + + # to LUB + normal[min_y:max_y+1, min_x:max_x+1, :] = roi_normal + normal = normal.astype(np.float32) + normal[:,:,0] *= -1 + normal[:,:,1] *= -1 + + # Make mask + roi_mask = np.logical_or(np.logical_or(roi_normal[:,:,0] != 0, roi_normal[:,:,1] != 0), roi_normal[:,:,2] != 0).astype(np.float32) + mask[min_y:max_y+1, min_x:max_x+1] = roi_mask + mask = mask[:, :, None] + mask = mask > 0.5 + + return normal, mask + + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "val/100277_DT_img.png" + scene_name = sample_path.split('/')[0] + img_name, img_ext = sample_path.split('/')[-1].split('_img') + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'oasis') + img_path = '%s/%s' % (dataset_path, sample_path) + normal_path = img_path.replace('_img'+img_ext, '_normal.pkl') + intrins_path = img_path.replace('_img'+img_ext, '_intrins.npy') + assert os.path.exists(img_path) + assert os.path.exists(normal_path) + assert os.path.exists(intrins_path) + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + + # read normal (H, W, 3) + h = img.shape[0] + w = img.shape[1] + normal, normal_mask = read_normal(normal_path, h, w) + + # read intrins (3, 3) + intrins = np.load(intrins_path) + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='oasis', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample diff --git a/infer/dataset_normal/oasis/split/val.txt b/infer/dataset_normal/oasis/split/val.txt new file mode 100644 index 0000000000000000000000000000000000000000..bfcaa6e3a5dffff598354e55f2effda45f570e76 --- /dev/null +++ b/infer/dataset_normal/oasis/split/val.txt @@ -0,0 +1,6000 @@ +val/246011_ALI_img.png +val/246017_ALI_img.png +val/246020_ALI_img.png +val/246022_ALI_img.png +val/246027_ALI_img.png +val/246029_ALI_img.png +val/246037_ALI_img.png +val/246038_ALI_img.png +val/246043_ALI_img.png +val/246045_ALI_img.png +val/246046_ALI_img.png +val/246054_ALI_img.png +val/246073_ALI_img.png +val/246077_ALI_img.png +val/246089_ALI_img.png +val/246092_ALI_img.png +val/246100_ALI_img.png +val/246101_ALI_img.png +val/246102_ALI_img.png +val/246107_ALI_img.png +val/246110_ALI_img.png +val/246112_ALI_img.png +val/246117_ALI_img.png +val/246122_ALI_img.png +val/246124_ALI_img.png +val/246128_ALI_img.png +val/246130_ALI_img.png +val/246132_ALI_img.png +val/246133_ALI_img.png +val/246134_DT_img.png +val/246147_ALI_img.png +val/246149_ALI_img.png +val/246152_ALI_img.png +val/246155_ALI_img.png +val/246157_ALI_img.png +val/246162_ALI_img.png +val/246163_ALI_img.png +val/246168_ALI_img.png +val/246172_ALI_img.png +val/246173_ALI_img.png +val/246179_ALI_img.png +val/246184_ALI_img.png +val/246185_ALI_img.png +val/246186_ALI_img.png +val/246188_ALI_img.png +val/246189_ALI_img.png +val/246194_ALI_img.png +val/246205_ALI_img.png +val/246206_ALI_img.png +val/246207_ALI_img.png +val/246208_ALI_img.png +val/246212_ALI_img.png +val/246215_ALI_img.png +val/246216_ALI_img.png +val/246219_ALI_img.png +val/246224_ALI_img.png +val/246230_ALI_img.png +val/246232_ALI_img.png +val/246234_ALI_img.png +val/246241_ALI_img.png +val/246243_ALI_img.png +val/246244_ALI_img.png +val/246247_ALI_img.png +val/246249_ALI_img.png +val/246252_ALI_img.png +val/246255_ALI_img.png +val/246259_ALI_img.png +val/246264_ALI_img.png +val/246266_ALI_img.png +val/246269_ALI_img.png +val/246273_ALI_img.png +val/246274_ALI_img.png +val/246287_ALI_img.png +val/246290_ALI_img.png +val/246291_ALI_img.png +val/246294_ALI_img.png +val/246295_ALI_img.png +val/246296_ALI_img.png +val/246303_ALI_img.png +val/246311_ALI_img.png +val/246313_ALI_img.png +val/246315_ALI_img.png +val/246324_ALI_img.png +val/246325_DT_img.png +val/246326_ALI_img.png +val/246337_ALI_img.png +val/246342_ALI_img.png +val/246347_ALI_img.png +val/246349_ALI_img.png +val/246351_ALI_img.png +val/246354_ALI_img.png +val/246358_ALI_img.png +val/246366_ALI_img.png +val/246377_ALI_img.png +val/246382_ALI_img.png +val/246383_ALI_img.png +val/246386_ALI_img.png +val/246388_ALI_img.png +val/246389_ALI_img.png +val/246391_ALI_img.png +val/246396_ALI_img.png +val/246401_ALI_img.png +val/246402_ALI_img.png +val/246403_ALI_img.png +val/246404_ALI_img.png +val/246406_ALI_img.png +val/246409_ALI_img.png +val/246411_ALI_img.png +val/246413_ALI_img.png +val/246416_ALI_img.png +val/246424_ALI_img.png +val/246428_ALI_img.png +val/246429_ALI_img.png +val/246433_ALI_img.png +val/246434_ALI_img.png +val/246446_ALI_img.png +val/246447_ALI_img.png +val/246450_ALI_img.png +val/246452_ALI_img.png +val/246458_ALI_img.png +val/246468_ALI_img.png +val/246474_ALI_img.png +val/246477_ALI_img.png +val/246479_ALI_img.png +val/246481_ALI_img.png +val/246490_ALI_img.png +val/246492_ALI_img.png +val/246494_ALI_img.png +val/246500_ALI_img.png +val/246502_ALI_img.png +val/246506_ALI_img.png +val/246512_ALI_img.png +val/246513_ALI_img.png +val/246528_ALI_img.png +val/246531_ALI_img.png +val/246532_ALI_img.png +val/246536_ALI_img.png +val/246547_ALI_img.png +val/246551_ALI_img.png +val/246554_ALI_img.png +val/246557_ALI_img.png +val/246562_ALI_img.png +val/246572_ALI_img.png +val/246573_ALI_img.png +val/246574_ALI_img.png +val/246575_ALI_img.png +val/246576_ALI_img.png +val/246585_ALI_img.png +val/246598_ALI_img.png +val/246613_ALI_img.png +val/246616_ALI_img.png +val/246618_ALI_img.png +val/246621_ALI_img.png +val/246624_ALI_img.png +val/246630_ALI_img.png +val/246638_ALI_img.png +val/246648_ALI_img.png +val/246652_ALI_img.png +val/246655_ALI_img.png +val/246666_ALI_img.png +val/246671_ALI_img.png +val/246682_ALI_img.png +val/246687_ALI_img.png +val/246690_ALI_img.png +val/246691_ALI_img.png +val/246694_ALI_img.png +val/246696_ALI_img.png +val/246700_ALI_img.png +val/246703_ALI_img.png +val/246707_ALI_img.png +val/246712_ALI_img.png +val/246716_ALI_img.png +val/246719_ALI_img.png +val/246727_ALI_img.png +val/246730_ALI_img.png +val/246734_ALI_img.png +val/246735_ALI_img.png +val/246736_ALI_img.png +val/246738_ALI_img.png +val/246743_ALI_img.png +val/246750_ALI_img.png +val/246751_ALI_img.png +val/246754_ALI_img.png +val/246761_ALI_img.png +val/246773_ALI_img.png +val/246776_ALI_img.png +val/246777_ALI_img.png +val/246781_ALI_img.png +val/246784_ALI_img.png +val/246786_ALI_img.png +val/246787_ALI_img.png +val/246794_ALI_img.png +val/246799_ALI_img.png +val/246803_ALI_img.png +val/246805_ALI_img.png +val/246810_ALI_img.png +val/246811_ALI_img.png +val/246815_ALI_img.png +val/246823_ALI_img.png +val/246824_ALI_img.png +val/246839_ALI_img.png +val/246840_ALI_img.png +val/246842_ALI_img.png +val/246846_ALI_img.png +val/246848_ALI_img.png +val/246852_ALI_img.png +val/246854_ALI_img.png +val/246862_ALI_img.png +val/246866_ALI_img.png +val/246870_ALI_img.png +val/246872_ALI_img.png +val/246874_ALI_img.png +val/246876_ALI_img.png +val/246880_ALI_img.png +val/246882_ALI_img.png +val/246884_ALI_img.png +val/246898_ALI_img.png +val/246900_ALI_img.png +val/246901_ALI_img.png +val/246903_ALI_img.png +val/246905_ALI_img.png +val/246907_ALI_img.png +val/246918_ALI_img.png +val/246923_ALI_img.png +val/246935_ALI_img.png +val/246937_ALI_img.png +val/246938_ALI_img.png +val/246945_ALI_img.png +val/246949_ALI_img.png +val/246955_ALI_img.png +val/246968_ALI_img.png +val/246971_ALI_img.png +val/246972_ALI_img.png +val/246975_ALI_img.png +val/246976_ALI_img.png +val/246981_ALI_img.png +val/246983_ALI_img.png +val/246986_ALI_img.png +val/246991_ALI_img.png +val/246995_ALI_img.png +val/247003_ALI_img.png +val/247006_ALI_img.png +val/247007_ALI_img.png +val/247008_ALI_img.png +val/247009_ALI_img.png +val/247013_ALI_img.png +val/247015_ALI_img.png +val/247018_ALI_img.png +val/247019_ALI_img.png +val/247021_ALI_img.png +val/247022_ALI_img.png +val/247023_ALI_img.png +val/247025_ALI_img.png +val/247026_ALI_img.png +val/247030_ALI_img.png +val/247031_ALI_img.png +val/247034_ALI_img.png +val/247035_ALI_img.png +val/247037_ALI_img.png +val/247042_ALI_img.png +val/247050_ALI_img.png +val/247059_ALI_img.png +val/247060_ALI_img.png +val/247075_ALI_img.png +val/247077_ALI_img.png +val/247081_ALI_img.png +val/247082_ALI_img.png +val/247085_ALI_img.png +val/247087_ALI_img.png +val/247089_ALI_img.png +val/247092_ALI_img.png +val/247098_ALI_img.png +val/247102_ALI_img.png +val/247104_ALI_img.png +val/247110_ALI_img.png +val/247111_ALI_img.png +val/247114_ALI_img.png +val/247116_ALI_img.png +val/247117_ALI_img.png +val/247118_ALI_img.png +val/247121_ALI_img.png +val/247124_ALI_img.png +val/247126_ALI_img.png +val/247127_ALI_img.png +val/247130_ALI_img.png +val/247131_ALI_img.png +val/247141_ALI_img.png +val/247142_ALI_img.png +val/247143_ALI_img.png +val/247146_ALI_img.png +val/247152_ALI_img.png +val/247156_ALI_img.png +val/247157_ALI_img.png +val/247159_ALI_img.png +val/247167_ALI_img.png +val/247171_ALI_img.png +val/247177_ALI_img.png +val/247179_ALI_img.png +val/247182_ALI_img.png +val/247189_ALI_img.png +val/247192_ALI_img.png +val/247193_ALI_img.png +val/247203_ALI_img.png +val/247206_ALI_img.png +val/247209_ALI_img.png +val/247213_ALI_img.png +val/247214_ALI_img.png +val/247216_ALI_img.png +val/247217_ALI_img.png +val/247218_ALI_img.png +val/247224_ALI_img.png +val/247230_ALI_img.png +val/247237_ALI_img.png +val/247242_ALI_img.png +val/247245_ALI_img.png +val/247246_ALI_img.png +val/247248_ALI_img.png +val/247254_ALI_img.png +val/247258_ALI_img.png +val/247260_ALI_img.png +val/247261_ALI_img.png +val/247262_ALI_img.png +val/247264_ALI_img.png +val/247270_ALI_img.png +val/247273_ALI_img.png +val/247277_ALI_img.png +val/247279_ALI_img.png +val/247280_ALI_img.png +val/247281_ALI_img.png +val/247283_ALI_img.png +val/247284_ALI_img.png +val/247287_ALI_img.png +val/247289_ALI_img.png +val/247295_ALI_img.png +val/247298_ALI_img.png +val/247305_ALI_img.png +val/247309_ALI_img.png +val/247314_ALI_img.png +val/247320_ALI_img.png +val/247323_ALI_img.png +val/247327_ALI_img.png +val/247331_ALI_img.png +val/247333_ALI_img.png +val/247336_ALI_img.png +val/247346_ALI_img.png +val/247353_ALI_img.png +val/247355_ALI_img.png +val/247357_ALI_img.png +val/247359_ALI_img.png +val/247363_ALI_img.png +val/247386_ALI_img.png +val/247389_ALI_img.png +val/247397_ALI_img.png +val/247401_ALI_img.png +val/247409_ALI_img.png +val/247411_ALI_img.png +val/247413_ALI_img.png +val/247422_ALI_img.png +val/247424_ALI_img.png +val/247427_ALI_img.png +val/247428_ALI_img.png +val/247429_ALI_img.png +val/247431_ALI_img.png +val/247431_DT_img.png +val/247432_ALI_img.png +val/247438_ALI_img.png +val/247445_ALI_img.png +val/247446_ALI_img.png +val/247458_ALI_img.png +val/247462_ALI_img.png +val/247469_ALI_img.png +val/247474_ALI_img.png +val/247475_ALI_img.png +val/247476_ALI_img.png +val/247487_ALI_img.png +val/247494_ALI_img.png +val/247495_ALI_img.png +val/247496_ALI_img.png +val/247497_ALI_img.png +val/247498_ALI_img.png +val/247500_ALI_img.png +val/247509_ALI_img.png +val/247511_ALI_img.png +val/247512_ALI_img.png +val/247513_ALI_img.png +val/247514_ALI_img.png +val/247525_ALI_img.png +val/247534_ALI_img.png +val/247535_ALI_img.png +val/247538_ALI_img.png +val/247546_ALI_img.png +val/247551_ALI_img.png +val/247554_ALI_img.png +val/247555_ALI_img.png +val/247557_ALI_img.png +val/247562_ALI_img.png +val/247567_ALI_img.png +val/247569_ALI_img.png +val/247579_ALI_img.png +val/247586_ALI_img.png +val/247590_ALI_img.png +val/247591_ALI_img.png +val/247595_ALI_img.png +val/247600_ALI_img.png +val/247607_ALI_img.png +val/247611_ALI_img.png +val/247613_ALI_img.png +val/247614_ALI_img.png +val/247622_ALI_img.png +val/247624_ALI_img.png +val/247628_ALI_img.png +val/247634_ALI_img.png +val/247636_ALI_img.png +val/247638_ALI_img.png +val/247639_ALI_img.png +val/247640_ALI_img.png +val/247647_ALI_img.png +val/247655_ALI_img.png +val/247666_ALI_img.png +val/247667_ALI_img.png +val/247677_ALI_img.png +val/247682_ALI_img.png +val/247686_ALI_img.png +val/247690_ALI_img.png +val/247692_ALI_img.png +val/247693_ALI_img.png +val/247696_ALI_img.png +val/247700_ALI_img.png +val/247709_ALI_img.png +val/247710_ALI_img.png +val/247713_ALI_img.png +val/247715_ALI_img.png +val/247722_ALI_img.png +val/247730_ALI_img.png +val/247734_ALI_img.png +val/247741_ALI_img.png +val/247744_ALI_img.png +val/247748_ALI_img.png +val/247751_ALI_img.png +val/247752_ALI_img.png +val/247754_ALI_img.png +val/247764_ALI_img.png +val/247765_ALI_img.png +val/247766_ALI_img.png +val/247768_ALI_img.png +val/247774_ALI_img.png +val/247782_ALI_img.png +val/247789_ALI_img.png +val/247791_ALI_img.png +val/247794_ALI_img.png +val/247797_ALI_img.png +val/247798_ALI_img.png +val/247799_ALI_img.png +val/247802_ALI_img.png +val/247808_ALI_img.png +val/247809_ALI_img.png +val/247812_ALI_img.png +val/247815_ALI_img.png +val/247819_ALI_img.png +val/247821_ALI_img.png +val/247822_ALI_img.png +val/247823_ALI_img.png +val/247830_ALI_img.png +val/247834_ALI_img.png +val/247836_ALI_img.png +val/247837_ALI_img.png +val/247838_ALI_img.png +val/247839_ALI_img.png +val/247846_ALI_img.png +val/247848_ALI_img.png +val/247849_ALI_img.png +val/247850_ALI_img.png +val/247852_ALI_img.png +val/247855_ALI_img.png +val/247857_ALI_img.png +val/247864_ALI_img.png +val/247886_ALI_img.png +val/247893_ALI_img.png +val/247897_ALI_img.png +val/247899_ALI_img.png +val/247913_ALI_img.png +val/247914_ALI_img.png +val/247915_ALI_img.png +val/247916_ALI_img.png +val/247919_ALI_img.png +val/247927_ALI_img.png +val/247941_ALI_img.png +val/247944_ALI_img.png +val/247948_ALI_img.png +val/247952_ALI_img.png +val/247962_ALI_img.png +val/247968_ALI_img.png +val/247969_ALI_img.png +val/247974_ALI_img.png +val/247977_ALI_img.png +val/247978_ALI_img.png +val/247979_ALI_img.png +val/247985_ALI_img.png +val/247987_ALI_img.png +val/247991_ALI_img.png +val/247993_ALI_img.png +val/247995_ALI_img.png +val/247999_ALI_img.png +val/248001_ALI_img.png +val/248002_ALI_img.png +val/248009_ALI_img.png +val/248012_ALI_img.png +val/248023_ALI_img.png +val/248024_ALI_img.png +val/248028_ALI_img.png +val/248029_ALI_img.png +val/248031_ALI_img.png +val/248034_ALI_img.png +val/248037_ALI_img.png +val/248038_ALI_img.png +val/248046_ALI_img.png +val/248050_ALI_img.png +val/248052_ALI_img.png +val/248057_ALI_img.png +val/248061_ALI_img.png +val/248066_ALI_img.png +val/248068_ALI_img.png +val/248075_ALI_img.png +val/248076_ALI_img.png +val/248082_ALI_img.png +val/248083_ALI_img.png +val/248086_ALI_img.png +val/248091_ALI_img.png +val/248095_ALI_img.png +val/248096_ALI_img.png +val/248099_ALI_img.png +val/248104_ALI_img.png +val/248108_ALI_img.png +val/248110_ALI_img.png +val/248116_ALI_img.png +val/248120_ALI_img.png +val/248121_ALI_img.png +val/248123_ALI_img.png +val/248124_ALI_img.png +val/248126_ALI_img.png +val/248136_ALI_img.png +val/248138_ALI_img.png +val/248139_ALI_img.png +val/248151_ALI_img.png +val/248153_ALI_img.png +val/248157_ALI_img.png +val/248158_ALI_img.png +val/248159_ALI_img.png +val/248161_ALI_img.png +val/248168_ALI_img.png +val/248172_ALI_img.png +val/248182_ALI_img.png +val/248186_ALI_img.png +val/248190_ALI_img.png +val/248194_ALI_img.png +val/248196_ALI_img.png +val/248201_ALI_img.png +val/248204_ALI_img.png +val/248206_ALI_img.png +val/248210_ALI_img.png +val/248216_ALI_img.png +val/248220_ALI_img.png +val/248222_ALI_img.png +val/248224_ALI_img.png +val/248228_ALI_img.png +val/248229_ALI_img.png +val/248233_ALI_img.png +val/248235_ALI_img.png +val/248238_ALI_img.png +val/248239_ALI_img.png +val/248244_ALI_img.png +val/248247_ALI_img.png +val/248248_ALI_img.png +val/248253_ALI_img.png +val/248260_ALI_img.png +val/248264_ALI_img.png +val/248267_ALI_img.png +val/248268_ALI_img.png +val/248269_ALI_img.png +val/248275_ALI_img.png +val/248278_ALI_img.png +val/248282_ALI_img.png +val/248290_ALI_img.png +val/248291_ALI_img.png +val/248292_ALI_img.png +val/248297_ALI_img.png +val/248301_ALI_img.png +val/248303_ALI_img.png +val/248304_ALI_img.png +val/248306_ALI_img.png +val/248310_ALI_img.png +val/248311_ALI_img.png +val/248320_ALI_img.png +val/248329_ALI_img.png +val/248331_ALI_img.png +val/248338_ALI_img.png +val/248339_ALI_img.png +val/248340_ALI_img.png +val/248341_ALI_img.png +val/248343_ALI_img.png +val/248349_ALI_img.png +val/248354_ALI_img.png +val/248356_ALI_img.png +val/248360_ALI_img.png +val/248361_ALI_img.png +val/248363_ALI_img.png +val/248366_ALI_img.png +val/248371_ALI_img.png +val/248372_ALI_img.png +val/248377_ALI_img.png +val/248385_ALI_img.png +val/248392_ALI_img.png +val/248396_ALI_img.png +val/248399_ALI_img.png +val/248402_ALI_img.png +val/248403_ALI_img.png +val/248408_ALI_img.png +val/248411_ALI_img.png +val/248415_ALI_img.png +val/248416_ALI_img.png +val/248419_ALI_img.png +val/248429_ALI_img.png +val/248431_ALI_img.png +val/248433_ALI_img.png +val/248434_ALI_img.png +val/248436_ALI_img.png +val/248437_ALI_img.png +val/248438_ALI_img.png +val/248447_ALI_img.png +val/248452_ALI_img.png +val/248455_ALI_img.png +val/248459_ALI_img.png +val/248460_ALI_img.png +val/248465_ALI_img.png +val/248468_ALI_img.png +val/248472_ALI_img.png +val/248476_ALI_img.png +val/248484_ALI_img.png +val/248489_ALI_img.png +val/248493_ALI_img.png +val/248497_ALI_img.png +val/248498_ALI_img.png +val/248501_ALI_img.png +val/248505_ALI_img.png +val/248509_ALI_img.png +val/248515_DT_img.png +val/248516_ALI_img.png +val/248517_ALI_img.png +val/248519_ALI_img.png +val/248529_ALI_img.png +val/248530_ALI_img.png +val/248537_ALI_img.png +val/248538_ALI_img.png +val/248546_ALI_img.png +val/248549_ALI_img.png +val/248551_ALI_img.png +val/248558_ALI_img.png +val/248566_ALI_img.png +val/248573_ALI_img.png +val/248575_ALI_img.png +val/248581_ALI_img.png +val/248585_ALI_img.png +val/248587_ALI_img.png +val/248589_ALI_img.png +val/248591_ALI_img.png +val/248595_ALI_img.png +val/248601_ALI_img.png +val/248603_ALI_img.png +val/248615_ALI_img.png +val/248616_ALI_img.png +val/248617_ALI_img.png +val/248623_ALI_img.png +val/248624_ALI_img.png +val/248627_ALI_img.png +val/248628_ALI_img.png +val/248630_ALI_img.png +val/248631_ALI_img.png +val/248632_ALI_img.png +val/248637_ALI_img.png +val/248644_ALI_img.png +val/248649_ALI_img.png +val/248650_ALI_img.png +val/248653_ALI_img.png +val/248656_ALI_img.png +val/248658_ALI_img.png +val/248661_ALI_img.png +val/248667_ALI_img.png +val/248668_ALI_img.png +val/248677_ALI_img.png +val/248678_ALI_img.png +val/248680_ALI_img.png +val/248686_ALI_img.png +val/248690_ALI_img.png +val/248691_ALI_img.png +val/248694_ALI_img.png +val/248695_ALI_img.png +val/248701_ALI_img.png +val/248704_ALI_img.png +val/248706_ALI_img.png +val/248707_ALI_img.png +val/248714_ALI_img.png +val/248718_ALI_img.png +val/248722_ALI_img.png +val/248725_ALI_img.png +val/248727_ALI_img.png +val/248729_ALI_img.png +val/248731_ALI_img.png +val/248735_ALI_img.png +val/248736_ALI_img.png +val/248737_ALI_img.png +val/248742_ALI_img.png +val/248743_ALI_img.png +val/248746_ALI_img.png +val/248752_ALI_img.png +val/248757_ALI_img.png +val/248758_ALI_img.png +val/248762_ALI_img.png +val/248763_ALI_img.png +val/248776_ALI_img.png +val/248778_ALI_img.png +val/248792_ALI_img.png +val/248793_ALI_img.png +val/248801_ALI_img.png +val/248811_ALI_img.png +val/248815_ALI_img.png +val/248816_ALI_img.png +val/248817_ALI_img.png +val/248818_ALI_img.png +val/248819_ALI_img.png +val/248820_ALI_img.png +val/248822_ALI_img.png +val/248823_ALI_img.png +val/248824_ALI_img.png +val/248826_ALI_img.png +val/248828_ALI_img.png +val/248835_ALI_img.png +val/248836_ALI_img.png +val/248839_ALI_img.png +val/248840_ALI_img.png +val/248844_ALI_img.png +val/248846_ALI_img.png +val/248847_ALI_img.png +val/248850_ALI_img.png +val/248861_ALI_img.png +val/248863_ALI_img.png +val/248864_ALI_img.png +val/248866_ALI_img.png +val/248869_ALI_img.png +val/248870_ALI_img.png +val/248874_ALI_img.png +val/248882_ALI_img.png +val/248888_ALI_img.png +val/248897_ALI_img.png +val/248902_ALI_img.png +val/248911_ALI_img.png +val/248923_ALI_img.png +val/248926_ALI_img.png +val/248927_ALI_img.png +val/248928_ALI_img.png +val/248929_ALI_img.png +val/248929_DT_img.png +val/248932_ALI_img.png +val/248933_ALI_img.png +val/248937_ALI_img.png +val/248938_ALI_img.png +val/248940_ALI_img.png +val/248942_ALI_img.png +val/248944_ALI_img.png +val/248956_ALI_img.png +val/248963_ALI_img.png +val/248966_ALI_img.png +val/248970_ALI_img.png +val/248979_ALI_img.png +val/248980_ALI_img.png +val/248982_ALI_img.png +val/248987_ALI_img.png +val/249002_ALI_img.png +val/249004_ALI_img.png +val/249007_ALI_img.png +val/249013_ALI_img.png +val/249015_ALI_img.png +val/249023_ALI_img.png +val/249027_ALI_img.png +val/249034_ALI_img.png +val/249038_ALI_img.png +val/249044_ALI_img.png +val/249048_ALI_img.png +val/249050_ALI_img.png +val/249064_ALI_img.png +val/249069_ALI_img.png +val/249075_ALI_img.png +val/249079_ALI_img.png +val/249081_ALI_img.png +val/249083_ALI_img.png +val/249093_ALI_img.png +val/249096_ALI_img.png +val/249100_ALI_img.png +val/249103_ALI_img.png +val/249104_ALI_img.png +val/249109_ALI_img.png +val/249110_ALI_img.png +val/249111_ALI_img.png +val/249122_ALI_img.png +val/249123_ALI_img.png +val/249125_ALI_img.png +val/249127_ALI_img.png +val/249128_ALI_img.png +val/249135_ALI_img.png +val/249144_ALI_img.png +val/249146_ALI_img.png +val/249149_ALI_img.png +val/249165_ALI_img.png +val/249169_ALI_img.png +val/249172_ALI_img.png +val/249179_ALI_img.png +val/249184_ALI_img.png +val/249186_ALI_img.png +val/249191_ALI_img.png +val/249193_ALI_img.png +val/249199_ALI_img.png +val/249200_ALI_img.png +val/249202_ALI_img.png +val/249211_ALI_img.png +val/249213_ALI_img.png +val/249219_ALI_img.png +val/249220_ALI_img.png +val/249228_ALI_img.png +val/249231_ALI_img.png +val/249233_ALI_img.png +val/249237_ALI_img.png +val/249248_ALI_img.png +val/249254_ALI_img.png +val/249255_ALI_img.png +val/249264_ALI_img.png +val/249267_ALI_img.png +val/249268_ALI_img.png +val/249270_ALI_img.png +val/249272_ALI_img.png +val/249273_ALI_img.png +val/249288_ALI_img.png +val/249289_ALI_img.png +val/249292_ALI_img.png +val/249293_ALI_img.png +val/249302_ALI_img.png +val/249306_ALI_img.png +val/249308_ALI_img.png +val/249310_ALI_img.png +val/249314_ALI_img.png +val/249316_ALI_img.png +val/249320_ALI_img.png +val/249321_ALI_img.png +val/249326_ALI_img.png +val/249328_ALI_img.png +val/249329_ALI_img.png +val/249330_ALI_img.png +val/249335_ALI_img.png +val/249342_ALI_img.png +val/249350_ALI_img.png +val/249354_ALI_img.png +val/249355_ALI_img.png +val/249357_ALI_img.png +val/249358_ALI_img.png +val/249368_ALI_img.png +val/249369_ALI_img.png +val/249372_ALI_img.png +val/249375_ALI_img.png +val/249378_ALI_img.png +val/249389_ALI_img.png +val/249396_ALI_img.png +val/249399_ALI_img.png +val/249402_ALI_img.png +val/249411_ALI_img.png +val/249412_ALI_img.png +val/249414_ALI_img.png +val/249416_ALI_img.png +val/249425_ALI_img.png +val/249426_ALI_img.png +val/249435_ALI_img.png +val/249439_ALI_img.png +val/249444_ALI_img.png +val/249447_ALI_img.png +val/249452_ALI_img.png +val/249456_ALI_img.png +val/249459_ALI_img.png +val/249465_ALI_img.png +val/249468_ALI_img.png +val/249477_ALI_img.png +val/249478_ALI_img.png +val/249482_ALI_img.png +val/249483_ALI_img.png +val/249486_ALI_img.png +val/249487_ALI_img.png +val/249493_ALI_img.png +val/249495_ALI_img.png +val/249498_ALI_img.png +val/249513_ALI_img.png +val/249515_ALI_img.png +val/249518_ALI_img.png +val/249519_ALI_img.png +val/249528_ALI_img.png +val/249531_ALI_img.png +val/249538_ALI_img.png +val/249540_ALI_img.png +val/249544_ALI_img.png +val/249545_ALI_img.png +val/249547_ALI_img.png +val/249550_ALI_img.png +val/249551_ALI_img.png +val/249552_ALI_img.png +val/249557_ALI_img.png +val/249559_ALI_img.png +val/249565_ALI_img.png +val/249572_ALI_img.png +val/249574_ALI_img.png +val/249587_ALI_img.png +val/249593_ALI_img.png +val/249603_ALI_img.png +val/249606_ALI_img.png +val/249608_ALI_img.png +val/249611_ALI_img.png +val/249623_ALI_img.png +val/249624_ALI_img.png +val/249637_ALI_img.png +val/249638_ALI_img.png +val/249640_ALI_img.png +val/249641_ALI_img.png +val/249651_ALI_img.png +val/249655_ALI_img.png +val/249659_ALI_img.png +val/249661_ALI_img.png +val/249666_ALI_img.png +val/249674_ALI_img.png +val/249675_ALI_img.png +val/249677_ALI_img.png +val/249679_ALI_img.png +val/249680_ALI_img.png +val/249682_ALI_img.png +val/249683_ALI_img.png +val/249689_ALI_img.png +val/249690_ALI_img.png +val/249691_ALI_img.png +val/249693_ALI_img.png +val/249698_ALI_img.png +val/249701_ALI_img.png +val/249704_ALI_img.png +val/249706_ALI_img.png +val/249707_ALI_img.png +val/249712_ALI_img.png +val/249714_ALI_img.png +val/249717_ALI_img.png +val/249718_ALI_img.png +val/249719_ALI_img.png +val/249720_ALI_img.png +val/249725_ALI_img.png +val/249727_ALI_img.png +val/249731_ALI_img.png +val/249748_ALI_img.png +val/249752_ALI_img.png +val/249758_ALI_img.png +val/249764_ALI_img.png +val/249765_ALI_img.png +val/249766_ALI_img.png +val/249768_ALI_img.png +val/249770_ALI_img.png +val/249772_ALI_img.png +val/249775_ALI_img.png +val/249778_ALI_img.png +val/249779_ALI_img.png +val/249782_ALI_img.png +val/249783_ALI_img.png +val/249788_ALI_img.png +val/249791_ALI_img.png +val/249800_ALI_img.png +val/249803_ALI_img.png +val/249806_ALI_img.png +val/249813_ALI_img.png +val/249829_ALI_img.png +val/249831_ALI_img.png +val/249833_ALI_img.png +val/249834_ALI_img.png +val/249835_ALI_img.png +val/249837_ALI_img.png +val/249840_ALI_img.png +val/249841_ALI_img.png +val/249846_ALI_img.png +val/249848_ALI_img.png +val/249856_ALI_img.png +val/249862_ALI_img.png +val/249866_ALI_img.png +val/249868_ALI_img.png +val/249871_ALI_img.png +val/249874_ALI_img.png +val/249877_ALI_img.png +val/249881_ALI_img.png +val/249883_ALI_img.png +val/249886_ALI_img.png +val/249888_ALI_img.png +val/249896_ALI_img.png +val/249897_ALI_img.png +val/249900_ALI_img.png +val/249901_ALI_img.png +val/249913_ALI_img.png +val/249914_ALI_img.png +val/249920_ALI_img.png +val/249927_ALI_img.png +val/249930_ALI_img.png +val/249933_ALI_img.png +val/249937_ALI_img.png +val/249945_ALI_img.png +val/249946_ALI_img.png +val/249949_ALI_img.png +val/249954_ALI_img.png +val/249962_ALI_img.png +val/249963_ALI_img.png +val/249964_ALI_img.png +val/249969_ALI_img.png +val/249977_ALI_img.png +val/249978_ALI_img.png +val/249986_ALI_img.png +val/249995_ALI_img.png +val/249998_ALI_img.png +val/250000_ALI_img.png +val/250003_ALI_img.png +val/250009_ALI_img.png +val/250019_ALI_img.png +val/250020_ALI_img.png +val/250022_ALI_img.png +val/250026_ALI_img.png +val/250040_ALI_img.png +val/250041_ALI_img.png +val/250042_ALI_img.png +val/250045_ALI_img.png +val/250056_ALI_img.png +val/250060_ALI_img.png +val/250062_ALI_img.png +val/250067_ALI_img.png +val/250068_ALI_img.png +val/250070_ALI_img.png +val/250071_DT_img.png +val/250076_ALI_img.png +val/250077_ALI_img.png +val/250080_ALI_img.png +val/250083_ALI_img.png +val/250084_ALI_img.png +val/250085_ALI_img.png +val/250087_ALI_img.png +val/250093_ALI_img.png +val/250095_ALI_img.png +val/250099_ALI_img.png +val/250115_ALI_img.png +val/250118_ALI_img.png +val/250124_ALI_img.png +val/250125_ALI_img.png +val/250126_ALI_img.png +val/250130_ALI_img.png +val/250132_ALI_img.png +val/250140_ALI_img.png +val/250144_ALI_img.png +val/250145_ALI_img.png +val/250153_ALI_img.png +val/250155_ALI_img.png +val/250157_ALI_img.png +val/250168_ALI_img.png +val/250174_ALI_img.png +val/250186_ALI_img.png +val/250190_ALI_img.png +val/250195_ALI_img.png +val/250198_ALI_img.png +val/250200_ALI_img.png +val/250202_ALI_img.png +val/250204_ALI_img.png +val/250205_ALI_img.png +val/250210_ALI_img.png +val/250211_ALI_img.png +val/250214_ALI_img.png +val/250215_ALI_img.png +val/250218_ALI_img.png +val/250219_ALI_img.png +val/250221_ALI_img.png +val/250224_ALI_img.png +val/250225_ALI_img.png +val/250230_ALI_img.png +val/250232_ALI_img.png +val/250237_ALI_img.png +val/250241_ALI_img.png +val/250242_ALI_img.png +val/250244_ALI_img.png +val/250245_ALI_img.png +val/250255_ALI_img.png +val/250259_ALI_img.png +val/250260_ALI_img.png +val/250262_ALI_img.png +val/250263_ALI_img.png +val/250268_ALI_img.png +val/250282_ALI_img.png +val/250290_ALI_img.png +val/250296_ALI_img.png +val/250297_ALI_img.png +val/250306_ALI_img.png +val/250310_ALI_img.png +val/250316_ALI_img.png +val/250317_ALI_img.png +val/250319_ALI_img.png +val/250321_ALI_img.png +val/250323_ALI_img.png +val/250328_ALI_img.png +val/250345_ALI_img.png +val/250348_ALI_img.png +val/250354_ALI_img.png +val/250356_ALI_img.png +val/250371_ALI_img.png +val/250379_ALI_img.png +val/250384_ALI_img.png +val/250388_ALI_img.png +val/250389_ALI_img.png +val/250390_ALI_img.png +val/250393_ALI_img.png +val/250395_ALI_img.png +val/250396_ALI_img.png +val/250397_ALI_img.png +val/250404_ALI_img.png +val/250414_ALI_img.png +val/250429_ALI_img.png +val/250435_ALI_img.png +val/250436_ALI_img.png +val/250438_ALI_img.png +val/250439_ALI_img.png +val/250446_ALI_img.png +val/250452_ALI_img.png +val/250454_ALI_img.png +val/250459_ALI_img.png +val/250461_ALI_img.png +val/250466_ALI_img.png +val/250468_ALI_img.png +val/250469_ALI_img.png +val/250475_ALI_img.png +val/250478_ALI_img.png +val/250479_ALI_img.png +val/250482_ALI_img.png +val/250493_ALI_img.png +val/250496_ALI_img.png +val/250499_ALI_img.png +val/250506_ALI_img.png +val/250507_ALI_img.png +val/250509_ALI_img.png +val/250510_ALI_img.png +val/250511_ALI_img.png +val/250522_ALI_img.png +val/250529_ALI_img.png +val/250537_ALI_img.png +val/250538_ALI_img.png +val/250553_ALI_img.png +val/250561_ALI_img.png +val/250564_ALI_img.png +val/250566_ALI_img.png +val/250568_ALI_img.png +val/250569_ALI_img.png +val/250570_ALI_img.png +val/250576_ALI_img.png +val/250579_ALI_img.png +val/250587_ALI_img.png +val/250588_ALI_img.png +val/250589_ALI_img.png +val/250590_ALI_img.png +val/250592_ALI_img.png +val/250596_ALI_img.png +val/250609_ALI_img.png +val/250610_ALI_img.png +val/250613_ALI_img.png +val/250616_ALI_img.png +val/250617_ALI_img.png +val/250623_ALI_img.png +val/250628_ALI_img.png +val/250630_ALI_img.png +val/250633_ALI_img.png +val/250634_ALI_img.png +val/250635_ALI_img.png +val/250636_ALI_img.png +val/250638_ALI_img.png +val/250639_ALI_img.png +val/250647_ALI_img.png +val/250649_ALI_img.png +val/250650_ALI_img.png +val/250652_ALI_img.png +val/250656_ALI_img.png +val/250657_ALI_img.png +val/250659_ALI_img.png +val/250664_ALI_img.png +val/250668_ALI_img.png +val/250669_ALI_img.png +val/250670_ALI_img.png +val/250674_ALI_img.png +val/250680_ALI_img.png +val/250686_ALI_img.png +val/250688_ALI_img.png +val/250701_ALI_img.png +val/250707_ALI_img.png +val/250711_ALI_img.png +val/250715_ALI_img.png +val/250721_ALI_img.png +val/250723_ALI_img.png +val/250727_ALI_img.png +val/250728_ALI_img.png +val/250730_ALI_img.png +val/250736_ALI_img.png +val/250740_ALI_img.png +val/250742_ALI_img.png +val/250744_ALI_img.png +val/250754_ALI_img.png +val/250762_ALI_img.png +val/250766_ALI_img.png +val/250767_ALI_img.png +val/250768_ALI_img.png +val/250770_ALI_img.png +val/250772_ALI_img.png +val/250775_ALI_img.png +val/250782_ALI_img.png +val/250786_ALI_img.png +val/250787_ALI_img.png +val/250791_ALI_img.png +val/250793_ALI_img.png +val/250795_ALI_img.png +val/250796_ALI_img.png +val/250797_ALI_img.png +val/250798_ALI_img.png +val/250807_ALI_img.png +val/250814_ALI_img.png +val/250821_ALI_img.png +val/250823_ALI_img.png +val/250830_ALI_img.png +val/250835_ALI_img.png +val/250836_ALI_img.png +val/250840_ALI_img.png +val/250841_ALI_img.png +val/250843_ALI_img.png +val/250850_ALI_img.png +val/250854_ALI_img.png +val/250855_ALI_img.png +val/250859_ALI_img.png +val/250860_ALI_img.png +val/250865_ALI_img.png +val/250866_ALI_img.png +val/250868_ALI_img.png +val/250872_ALI_img.png +val/250880_ALI_img.png +val/250883_ALI_img.png +val/250891_ALI_img.png +val/250895_ALI_img.png +val/250897_ALI_img.png +val/250900_ALI_img.png +val/250901_ALI_img.png +val/250902_ALI_img.png +val/250906_ALI_img.png +val/250913_ALI_img.png +val/250919_ALI_img.png +val/250921_ALI_img.png +val/250924_ALI_img.png +val/250936_ALI_img.png +val/250946_ALI_img.png +val/250947_ALI_img.png +val/250952_ALI_img.png +val/250953_ALI_img.png +val/250956_ALI_img.png +val/250957_ALI_img.png +val/250959_ALI_img.png +val/250973_ALI_img.png +val/250974_ALI_img.png +val/250978_ALI_img.png +val/250984_ALI_img.png +val/250985_ALI_img.png +val/250995_ALI_img.png +val/250998_ALI_img.png +val/250999_ALI_img.png +val/251001_ALI_img.png +val/251002_ALI_img.png +val/251003_ALI_img.png +val/251004_ALI_img.png +val/251014_ALI_img.png +val/251018_ALI_img.png +val/251025_ALI_img.png +val/251026_ALI_img.png +val/251033_ALI_img.png +val/251040_ALI_img.png +val/251051_ALI_img.png +val/251058_ALI_img.png +val/251063_ALI_img.png +val/251064_ALI_img.png +val/251072_ALI_img.png +val/251076_ALI_img.png +val/251078_ALI_img.png +val/251085_ALI_img.png +val/251086_ALI_img.png +val/251090_ALI_img.png +val/251091_ALI_img.png +val/251092_ALI_img.png +val/251098_ALI_img.png +val/251100_ALI_img.png +val/251107_ALI_img.png +val/251109_ALI_img.png +val/251117_ALI_img.png +val/251119_ALI_img.png +val/251123_ALI_img.png +val/251124_ALI_img.png +val/251126_ALI_img.png +val/251130_ALI_img.png +val/251134_ALI_img.png +val/251136_ALI_img.png +val/251142_ALI_img.png +val/251145_ALI_img.png +val/251148_ALI_img.png +val/251149_ALI_img.png +val/251152_ALI_img.png +val/251154_ALI_img.png +val/251159_ALI_img.png +val/251163_ALI_img.png +val/251164_ALI_img.png +val/251168_ALI_img.png +val/251172_ALI_img.png +val/251175_ALI_img.png +val/251179_ALI_img.png +val/251180_ALI_img.png +val/251183_ALI_img.png +val/251187_ALI_img.png +val/251189_ALI_img.png +val/251193_ALI_img.png +val/251194_ALI_img.png +val/251200_ALI_img.png +val/251201_ALI_img.png +val/251204_ALI_img.png +val/251209_ALI_img.png +val/251212_ALI_img.png +val/251213_ALI_img.png +val/251214_ALI_img.png +val/251215_ALI_img.png +val/251223_ALI_img.png +val/251234_ALI_img.png +val/251240_ALI_img.png +val/251244_ALI_img.png +val/251251_ALI_img.png +val/251252_ALI_img.png +val/251253_ALI_img.png +val/251254_ALI_img.png +val/251265_ALI_img.png +val/251266_ALI_img.png +val/251274_ALI_img.png +val/251276_ALI_img.png +val/251277_ALI_img.png +val/251279_ALI_img.png +val/251285_ALI_img.png +val/251287_ALI_img.png +val/251290_ALI_img.png +val/251293_ALI_img.png +val/251298_ALI_img.png +val/251299_ALI_img.png +val/251300_ALI_img.png +val/251321_ALI_img.png +val/251323_ALI_img.png +val/251334_ALI_img.png +val/251344_ALI_img.png +val/251344_DT_img.png +val/251345_ALI_img.png +val/251346_ALI_img.png +val/251349_ALI_img.png +val/251353_ALI_img.png +val/251356_ALI_img.png +val/251358_ALI_img.png +val/251366_ALI_img.png +val/251368_ALI_img.png +val/251370_ALI_img.png +val/251379_ALI_img.png +val/251380_ALI_img.png +val/251381_ALI_img.png +val/251383_ALI_img.png +val/251385_ALI_img.png +val/251388_ALI_img.png +val/251396_ALI_img.png +val/251398_ALI_img.png +val/251407_ALI_img.png +val/251408_ALI_img.png +val/251411_ALI_img.png +val/251413_ALI_img.png +val/251414_ALI_img.png +val/251419_ALI_img.png +val/251426_ALI_img.png +val/251429_ALI_img.png +val/251432_ALI_img.png +val/251436_ALI_img.png +val/251441_ALI_img.png +val/251445_ALI_img.png +val/251448_ALI_img.png +val/251452_ALI_img.png +val/251455_ALI_img.png +val/251457_ALI_img.png +val/251458_ALI_img.png +val/251461_ALI_img.png +val/251467_ALI_img.png +val/251468_ALI_img.png +val/251469_ALI_img.png +val/251471_ALI_img.png +val/251478_ALI_img.png +val/251491_ALI_img.png +val/251496_ALI_img.png +val/251501_ALI_img.png +val/251503_ALI_img.png +val/251505_ALI_img.png +val/251509_ALI_img.png +val/251510_ALI_img.png +val/251512_ALI_img.png +val/251515_ALI_img.png +val/251516_ALI_img.png +val/251519_ALI_img.png +val/251528_ALI_img.png +val/251530_ALI_img.png +val/251534_ALI_img.png +val/251536_ALI_img.png +val/251542_ALI_img.png +val/251549_ALI_img.png +val/251550_ALI_img.png +val/251551_ALI_img.png +val/251555_ALI_img.png +val/251557_ALI_img.png +val/251558_ALI_img.png +val/251559_ALI_img.png +val/251566_ALI_img.png +val/251574_ALI_img.png +val/251575_ALI_img.png +val/251577_ALI_img.png +val/251579_ALI_img.png +val/251596_ALI_img.png +val/251606_ALI_img.png +val/251609_ALI_img.png +val/251611_ALI_img.png +val/251618_ALI_img.png +val/251619_ALI_img.png +val/251628_ALI_img.png +val/251636_ALI_img.png +val/251643_ALI_img.png +val/251647_ALI_img.png +val/251650_ALI_img.png +val/251654_ALI_img.png +val/251663_ALI_img.png +val/251670_ALI_img.png +val/251672_ALI_img.png +val/251676_ALI_img.png +val/251677_ALI_img.png +val/251684_ALI_img.png +val/251685_ALI_img.png +val/251688_ALI_img.png +val/251689_ALI_img.png +val/251697_ALI_img.png +val/251699_ALI_img.png +val/251710_ALI_img.png +val/251711_ALI_img.png +val/251719_ALI_img.png +val/251724_ALI_img.png +val/251729_ALI_img.png +val/251733_ALI_img.png +val/251734_ALI_img.png +val/251736_ALI_img.png +val/251739_ALI_img.png +val/251748_ALI_img.png +val/251749_ALI_img.png +val/251756_ALI_img.png +val/251758_ALI_img.png +val/251766_ALI_img.png +val/251773_ALI_img.png +val/251778_ALI_img.png +val/251781_ALI_img.png +val/251784_ALI_img.png +val/251795_ALI_img.png +val/251798_ALI_img.png +val/251799_ALI_img.png +val/251801_ALI_img.png +val/251806_ALI_img.png +val/251809_ALI_img.png +val/251811_ALI_img.png +val/251813_ALI_img.png +val/251825_ALI_img.png +val/251826_ALI_img.png +val/251827_ALI_img.png +val/251830_ALI_img.png +val/251836_ALI_img.png +val/251838_ALI_img.png +val/251839_ALI_img.png +val/251848_ALI_img.png +val/251852_ALI_img.png +val/251864_ALI_img.png +val/251870_ALI_img.png +val/251882_ALI_img.png +val/251886_ALI_img.png +val/251895_ALI_img.png +val/251896_ALI_img.png +val/251896_DT_img.png +val/251897_ALI_img.png +val/251900_ALI_img.png +val/251908_ALI_img.png +val/251910_ALI_img.png +val/251911_ALI_img.png +val/251919_ALI_img.png +val/251922_ALI_img.png +val/251930_ALI_img.png +val/251934_ALI_img.png +val/251937_ALI_img.png +val/251938_ALI_img.png +val/251949_ALI_img.png +val/251955_ALI_img.png +val/251960_ALI_img.png +val/251961_ALI_img.png +val/251965_ALI_img.png +val/251967_ALI_img.png +val/251969_ALI_img.png +val/251970_ALI_img.png +val/251973_ALI_img.png +val/251977_ALI_img.png +val/251978_ALI_img.png +val/251982_ALI_img.png +val/251985_DT_img.png +val/251990_ALI_img.png +val/251992_ALI_img.png +val/251994_ALI_img.png +val/251995_ALI_img.png +val/251997_ALI_img.png +val/252004_DT_img.png +val/252005_ALI_img.png +val/252010_ALI_img.png +val/252019_ALI_img.png +val/252021_ALI_img.png +val/252026_ALI_img.png +val/252029_ALI_img.png +val/252030_ALI_img.png +val/252039_ALI_img.png +val/252040_ALI_img.png +val/252043_ALI_img.png +val/252048_ALI_img.png +val/252050_ALI_img.png +val/252054_ALI_img.png +val/252060_ALI_img.png +val/252061_ALI_img.png +val/252062_ALI_img.png +val/252070_ALI_img.png +val/252073_ALI_img.png +val/252077_ALI_img.png +val/252080_ALI_img.png +val/252081_ALI_img.png +val/252082_ALI_img.png +val/252083_ALI_img.png +val/252088_ALI_img.png +val/252094_ALI_img.png +val/252097_ALI_img.png +val/252099_ALI_img.png +val/252102_ALI_img.png +val/252105_ALI_img.png +val/252109_ALI_img.png +val/252120_ALI_img.png +val/252122_ALI_img.png +val/252124_ALI_img.png +val/252125_ALI_img.png +val/252127_ALI_img.png +val/252129_ALI_img.png +val/252132_ALI_img.png +val/252135_ALI_img.png +val/252140_ALI_img.png +val/252141_ALI_img.png +val/252143_ALI_img.png +val/252147_ALI_img.png +val/252151_ALI_img.png +val/252154_ALI_img.png +val/252159_ALI_img.png +val/252169_ALI_img.png +val/252170_ALI_img.png +val/252180_ALI_img.png +val/252183_ALI_img.png +val/252190_ALI_img.png +val/252197_ALI_img.png +val/252200_DT_img.png +val/252203_ALI_img.png +val/252206_ALI_img.png +val/252207_ALI_img.png +val/252209_ALI_img.png +val/252219_ALI_img.png +val/252222_ALI_img.png +val/252223_ALI_img.png +val/252229_ALI_img.png +val/252230_ALI_img.png +val/252232_ALI_img.png +val/252241_ALI_img.png +val/252253_ALI_img.png +val/252254_ALI_img.png +val/252257_ALI_img.png +val/252258_ALI_img.png +val/252265_ALI_img.png +val/252266_ALI_img.png +val/252267_ALI_img.png +val/252268_ALI_img.png +val/252274_ALI_img.png +val/252279_ALI_img.png +val/252284_ALI_img.png +val/252287_ALI_img.png +val/252288_ALI_img.png +val/252289_ALI_img.png +val/252290_ALI_img.png +val/252294_ALI_img.png +val/252295_ALI_img.png +val/252303_ALI_img.png +val/252308_ALI_img.png +val/252310_ALI_img.png +val/252313_ALI_img.png +val/252314_ALI_img.png +val/252317_ALI_img.png +val/252323_ALI_img.png +val/252324_ALI_img.png +val/252325_ALI_img.png +val/252326_ALI_img.png +val/252331_ALI_img.png +val/252332_ALI_img.png +val/252333_ALI_img.png +val/252338_ALI_img.png +val/252340_ALI_img.png +val/252347_ALI_img.png +val/252348_ALI_img.png +val/252360_ALI_img.png +val/252361_ALI_img.png +val/252372_ALI_img.png +val/252376_ALI_img.png +val/252378_ALI_img.png +val/252382_ALI_img.png +val/252386_ALI_img.png +val/252389_ALI_img.png +val/252392_ALI_img.png +val/252393_ALI_img.png +val/252413_ALI_img.png +val/252417_ALI_img.png +val/252422_ALI_img.png +val/252423_ALI_img.png +val/252427_ALI_img.png +val/252436_ALI_img.png +val/252438_ALI_img.png +val/252453_ALI_img.png +val/252454_ALI_img.png +val/252455_ALI_img.png +val/252466_ALI_img.png +val/252468_ALI_img.png +val/252472_ALI_img.png +val/252476_ALI_img.png +val/252486_ALI_img.png +val/252495_ALI_img.png +val/252499_ALI_img.png +val/252511_ALI_img.png +val/252512_ALI_img.png +val/252524_ALI_img.png +val/252527_ALI_img.png +val/252528_ALI_img.png +val/252529_ALI_img.png +val/252532_ALI_img.png +val/252534_ALI_img.png +val/252536_ALI_img.png +val/252538_ALI_img.png +val/252539_ALI_img.png +val/252541_ALI_img.png +val/252551_ALI_img.png +val/252554_ALI_img.png +val/252557_ALI_img.png +val/252558_ALI_img.png +val/252579_ALI_img.png +val/252586_ALI_img.png +val/252596_ALI_img.png +val/252599_ALI_img.png +val/252600_ALI_img.png +val/252607_ALI_img.png +val/252610_ALI_img.png +val/252612_ALI_img.png +val/252621_ALI_img.png +val/252624_ALI_img.png +val/252627_ALI_img.png +val/252628_ALI_img.png +val/252631_ALI_img.png +val/252632_ALI_img.png +val/252637_ALI_img.png +val/252643_ALI_img.png +val/252653_ALI_img.png +val/252662_ALI_img.png +val/252665_ALI_img.png +val/252678_ALI_img.png +val/252680_ALI_img.png +val/252684_ALI_img.png +val/252688_ALI_img.png +val/252689_ALI_img.png +val/252690_ALI_img.png +val/252693_ALI_img.png +val/252700_ALI_img.png +val/252702_ALI_img.png +val/252707_ALI_img.png +val/252712_ALI_img.png +val/252714_ALI_img.png +val/252723_ALI_img.png +val/252729_ALI_img.png +val/252735_ALI_img.png +val/252740_ALI_img.png +val/252743_ALI_img.png +val/252747_ALI_img.png +val/252748_ALI_img.png +val/252755_ALI_img.png +val/252756_ALI_img.png +val/252757_ALI_img.png +val/252758_ALI_img.png +val/252761_ALI_img.png +val/252762_ALI_img.png +val/252766_ALI_img.png +val/252770_ALI_img.png +val/252785_ALI_img.png +val/252786_ALI_img.png +val/252798_ALI_img.png +val/252800_ALI_img.png +val/252812_ALI_img.png +val/252814_ALI_img.png +val/252815_ALI_img.png +val/252818_ALI_img.png +val/252822_ALI_img.png +val/252824_ALI_img.png +val/252827_ALI_img.png +val/252829_ALI_img.png +val/252831_ALI_img.png +val/252834_ALI_img.png +val/252837_ALI_img.png +val/252840_ALI_img.png +val/252841_ALI_img.png +val/252848_ALI_img.png +val/252851_ALI_img.png +val/252853_ALI_img.png +val/252862_ALI_img.png +val/252872_ALI_img.png +val/252874_ALI_img.png +val/252876_ALI_img.png +val/252880_ALI_img.png +val/252883_ALI_img.png +val/252884_ALI_img.png +val/252892_ALI_img.png +val/252897_ALI_img.png +val/252902_ALI_img.png +val/252903_ALI_img.png +val/252904_ALI_img.png +val/252905_ALI_img.png +val/252914_ALI_img.png +val/252917_ALI_img.png +val/252918_ALI_img.png +val/252924_ALI_img.png +val/252930_ALI_img.png +val/252934_ALI_img.png +val/252942_ALI_img.png +val/252946_ALI_img.png +val/252955_ALI_img.png +val/252957_ALI_img.png +val/252958_ALI_img.png +val/252959_ALI_img.png +val/252960_ALI_img.png +val/252962_ALI_img.png +val/252968_ALI_img.png +val/252972_ALI_img.png +val/252976_ALI_img.png +val/252983_ALI_img.png +val/252986_ALI_img.png +val/252987_ALI_img.png +val/252988_ALI_img.png +val/252991_ALI_img.png +val/252997_ALI_img.png +val/253008_ALI_img.png +val/253011_ALI_img.png +val/253016_ALI_img.png +val/253023_ALI_img.png +val/253025_ALI_img.png +val/253029_ALI_img.png +val/253034_ALI_img.png +val/253039_ALI_img.png +val/253040_ALI_img.png +val/253051_ALI_img.png +val/253052_ALI_img.png +val/253056_ALI_img.png +val/253063_ALI_img.png +val/253064_ALI_img.png +val/253065_ALI_img.png +val/253070_ALI_img.png +val/253076_ALI_img.png +val/253078_DT_img.png +val/253079_ALI_img.png +val/253081_ALI_img.png +val/253088_ALI_img.png +val/253090_ALI_img.png +val/253094_ALI_img.png +val/253096_ALI_img.png +val/253099_ALI_img.png +val/253101_ALI_img.png +val/253103_ALI_img.png +val/253105_ALI_img.png +val/253107_ALI_img.png +val/253108_ALI_img.png +val/253110_ALI_img.png +val/253111_ALI_img.png +val/253116_ALI_img.png +val/253122_ALI_img.png +val/253128_ALI_img.png +val/253130_ALI_img.png +val/253133_ALI_img.png +val/253138_ALI_img.png +val/253142_ALI_img.png +val/253146_ALI_img.png +val/253148_ALI_img.png +val/253153_ALI_img.png +val/253159_ALI_img.png +val/253162_ALI_img.png +val/253172_ALI_img.png +val/253174_ALI_img.png +val/253183_ALI_img.png +val/253184_ALI_img.png +val/253186_ALI_img.png +val/253194_ALI_img.png +val/253196_ALI_img.png +val/253201_ALI_img.png +val/253209_ALI_img.png +val/253210_ALI_img.png +val/253217_ALI_img.png +val/253219_ALI_img.png +val/253221_ALI_img.png +val/253227_ALI_img.png +val/253229_ALI_img.png +val/253235_ALI_img.png +val/253241_ALI_img.png +val/253244_ALI_img.png +val/253248_ALI_img.png +val/253253_ALI_img.png +val/253256_ALI_img.png +val/253257_ALI_img.png +val/253258_ALI_img.png +val/253260_ALI_img.png +val/253263_ALI_img.png +val/253265_ALI_img.png +val/253270_ALI_img.png +val/253271_ALI_img.png +val/253275_ALI_img.png +val/253278_ALI_img.png +val/253279_ALI_img.png +val/253287_ALI_img.png +val/253289_ALI_img.png +val/253291_ALI_img.png +val/253296_ALI_img.png +val/253298_ALI_img.png +val/253305_ALI_img.png +val/253309_ALI_img.png +val/253313_ALI_img.png +val/253314_ALI_img.png +val/253316_ALI_img.png +val/253322_ALI_img.png +val/253326_ALI_img.png +val/253339_ALI_img.png +val/253345_ALI_img.png +val/253351_ALI_img.png +val/253353_ALI_img.png +val/253354_ALI_img.png +val/253355_ALI_img.png +val/253356_ALI_img.png +val/253358_ALI_img.png +val/253359_ALI_img.png +val/253362_ALI_img.png +val/253363_ALI_img.png +val/253369_ALI_img.png +val/253371_ALI_img.png +val/253372_ALI_img.png +val/253374_ALI_img.png +val/253375_ALI_img.png +val/253376_ALI_img.png +val/253379_ALI_img.png +val/253380_ALI_img.png +val/253383_ALI_img.png +val/253389_ALI_img.png +val/253394_ALI_img.png +val/253398_ALI_img.png +val/253399_ALI_img.png +val/253400_ALI_img.png +val/253401_ALI_img.png +val/253405_ALI_img.png +val/253409_ALI_img.png +val/253410_ALI_img.png +val/253412_ALI_img.png +val/253415_ALI_img.png +val/253416_ALI_img.png +val/253420_ALI_img.png +val/253430_ALI_img.png +val/253434_ALI_img.png +val/253439_ALI_img.png +val/253445_ALI_img.png +val/253448_ALI_img.png +val/253456_ALI_img.png +val/253460_ALI_img.png +val/253462_ALI_img.png +val/253465_ALI_img.png +val/253469_ALI_img.png +val/253471_ALI_img.png +val/253472_ALI_img.png +val/253478_ALI_img.png +val/253480_ALI_img.png +val/253481_ALI_img.png +val/253485_ALI_img.png +val/253486_ALI_img.png +val/253487_ALI_img.png +val/253493_ALI_img.png +val/253502_ALI_img.png +val/253505_ALI_img.png +val/253513_ALI_img.png +val/253515_ALI_img.png +val/253518_ALI_img.png +val/253519_ALI_img.png +val/253528_ALI_img.png +val/253529_ALI_img.png +val/253542_ALI_img.png +val/253550_ALI_img.png +val/253551_ALI_img.png +val/253552_ALI_img.png +val/253555_ALI_img.png +val/253558_ALI_img.png +val/253560_ALI_img.png +val/253566_ALI_img.png +val/253573_ALI_img.png +val/253575_ALI_img.png +val/253577_ALI_img.png +val/253580_ALI_img.png +val/253585_ALI_img.png +val/253586_ALI_img.png +val/253587_ALI_img.png +val/253593_ALI_img.png +val/253594_ALI_img.png +val/253595_ALI_img.png +val/253601_ALI_img.png +val/253602_ALI_img.png +val/253603_ALI_img.png +val/253605_ALI_img.png +val/253607_ALI_img.png +val/253610_ALI_img.png +val/253611_ALI_img.png +val/253613_ALI_img.png +val/253615_ALI_img.png +val/253617_ALI_img.png +val/253623_ALI_img.png +val/253628_ALI_img.png +val/253631_ALI_img.png +val/253634_ALI_img.png +val/253635_ALI_img.png +val/253636_ALI_img.png +val/253637_ALI_img.png +val/253638_ALI_img.png +val/253639_ALI_img.png +val/253657_ALI_img.png +val/253658_ALI_img.png +val/253660_ALI_img.png +val/253668_ALI_img.png +val/253675_ALI_img.png +val/253677_ALI_img.png +val/253678_ALI_img.png +val/253682_ALI_img.png +val/253685_ALI_img.png +val/253686_ALI_img.png +val/253691_ALI_img.png +val/253697_ALI_img.png +val/253703_ALI_img.png +val/253704_ALI_img.png +val/253705_ALI_img.png +val/253706_ALI_img.png +val/253707_ALI_img.png +val/253711_ALI_img.png +val/253712_ALI_img.png +val/253716_ALI_img.png +val/253718_ALI_img.png +val/253726_ALI_img.png +val/253732_ALI_img.png +val/253752_ALI_img.png +val/253753_DT_img.png +val/253755_ALI_img.png +val/253758_ALI_img.png +val/253762_ALI_img.png +val/253767_ALI_img.png +val/253771_ALI_img.png +val/253773_ALI_img.png +val/253775_ALI_img.png +val/253778_ALI_img.png +val/253781_ALI_img.png +val/253782_ALI_img.png +val/253785_ALI_img.png +val/253786_ALI_img.png +val/253787_ALI_img.png +val/253790_ALI_img.png +val/253792_ALI_img.png +val/253793_ALI_img.png +val/253796_ALI_img.png +val/253799_ALI_img.png +val/253801_ALI_img.png +val/253802_ALI_img.png +val/253803_ALI_img.png +val/253804_ALI_img.png +val/253806_ALI_img.png +val/253808_ALI_img.png +val/253811_ALI_img.png +val/253814_ALI_img.png +val/253817_ALI_img.png +val/253819_ALI_img.png +val/253820_ALI_img.png +val/253821_ALI_img.png +val/253824_ALI_img.png +val/253839_ALI_img.png +val/253844_ALI_img.png +val/253847_ALI_img.png +val/253849_ALI_img.png +val/253852_ALI_img.png +val/253855_ALI_img.png +val/253861_ALI_img.png +val/253864_ALI_img.png +val/253865_ALI_img.png +val/253869_ALI_img.png +val/253871_ALI_img.png +val/253872_ALI_img.png +val/253874_ALI_img.png +val/253877_ALI_img.png +val/253891_ALI_img.png +val/253892_ALI_img.png +val/253895_ALI_img.png +val/253897_ALI_img.png +val/253898_ALI_img.png +val/253909_ALI_img.png +val/253910_ALI_img.png +val/253912_ALI_img.png +val/253914_ALI_img.png +val/253919_ALI_img.png +val/253931_ALI_img.png +val/253932_ALI_img.png +val/253935_ALI_img.png +val/253942_ALI_img.png +val/253948_ALI_img.png +val/253960_ALI_img.png +val/253961_ALI_img.png +val/253964_ALI_img.png +val/253970_ALI_img.png +val/253972_ALI_img.png +val/253975_ALI_img.png +val/253977_ALI_img.png +val/253982_ALI_img.png +val/253985_ALI_img.png +val/253990_ALI_img.png +val/253992_ALI_img.png +val/254000_ALI_img.png +val/254007_ALI_img.png +val/254020_ALI_img.png +val/254024_ALI_img.png +val/254025_ALI_img.png +val/254028_ALI_img.png +val/254030_ALI_img.png +val/254033_ALI_img.png +val/254034_ALI_img.png +val/254038_ALI_img.png +val/254045_ALI_img.png +val/254047_ALI_img.png +val/254049_ALI_img.png +val/254050_ALI_img.png +val/254051_ALI_img.png +val/254057_ALI_img.png +val/254062_ALI_img.png +val/254066_ALI_img.png +val/254070_ALI_img.png +val/254094_ALI_img.png +val/254097_ALI_img.png +val/254102_ALI_img.png +val/254104_ALI_img.png +val/254106_ALI_img.png +val/254108_ALI_img.png +val/254110_ALI_img.png +val/254112_ALI_img.png +val/254117_ALI_img.png +val/254122_ALI_img.png +val/254124_ALI_img.png +val/254126_ALI_img.png +val/254127_ALI_img.png +val/254129_ALI_img.png +val/254130_ALI_img.png +val/254138_ALI_img.png +val/254141_ALI_img.png +val/254146_ALI_img.png +val/254147_ALI_img.png +val/254148_ALI_img.png +val/254151_ALI_img.png +val/254156_ALI_img.png +val/254157_ALI_img.png +val/254159_ALI_img.png +val/254161_ALI_img.png +val/254172_ALI_img.png +val/254175_ALI_img.png +val/254176_ALI_img.png +val/254189_ALI_img.png +val/254190_ALI_img.png +val/254194_ALI_img.png +val/254195_ALI_img.png +val/254210_ALI_img.png +val/254218_ALI_img.png +val/254226_ALI_img.png +val/254227_ALI_img.png +val/254231_ALI_img.png +val/254236_ALI_img.png +val/254237_ALI_img.png +val/254243_ALI_img.png +val/254248_ALI_img.png +val/254251_ALI_img.png +val/254253_ALI_img.png +val/254259_ALI_img.png +val/254268_ALI_img.png +val/254269_ALI_img.png +val/254273_ALI_img.png +val/254276_ALI_img.png +val/254277_ALI_img.png +val/254278_ALI_img.png +val/254280_ALI_img.png +val/254284_ALI_img.png +val/254292_ALI_img.png +val/254297_ALI_img.png +val/254302_ALI_img.png +val/254304_ALI_img.png +val/254307_ALI_img.png +val/254310_ALI_img.png +val/254313_ALI_img.png +val/254314_ALI_img.png +val/254316_ALI_img.png +val/254318_ALI_img.png +val/254319_ALI_img.png +val/254321_ALI_img.png +val/254324_ALI_img.png +val/254327_ALI_img.png +val/254340_ALI_img.png +val/254341_ALI_img.png +val/254350_ALI_img.png +val/254351_ALI_img.png +val/254354_ALI_img.png +val/254357_ALI_img.png +val/254361_ALI_img.png +val/254364_ALI_img.png +val/254368_ALI_img.png +val/254374_ALI_img.png +val/254388_ALI_img.png +val/254394_ALI_img.png +val/254395_ALI_img.png +val/254402_ALI_img.png +val/254403_ALI_img.png +val/254404_ALI_img.png +val/254405_ALI_img.png +val/254412_ALI_img.png +val/254416_ALI_img.png +val/254417_ALI_img.png +val/254421_ALI_img.png +val/254422_ALI_img.png +val/254424_ALI_img.png +val/254433_ALI_img.png +val/254446_ALI_img.png +val/254449_ALI_img.png +val/254452_ALI_img.png +val/254453_ALI_img.png +val/254456_ALI_img.png +val/254462_ALI_img.png +val/254467_ALI_img.png +val/254468_ALI_img.png +val/254469_ALI_img.png +val/254470_ALI_img.png +val/254472_ALI_img.png +val/254473_ALI_img.png +val/254479_ALI_img.png +val/254480_ALI_img.png +val/254481_ALI_img.png +val/254483_ALI_img.png +val/254484_ALI_img.png +val/254487_ALI_img.png +val/254490_ALI_img.png +val/254491_ALI_img.png +val/254499_ALI_img.png +val/254500_ALI_img.png +val/254502_ALI_img.png +val/254509_ALI_img.png +val/254511_ALI_img.png +val/254518_ALI_img.png +val/254519_ALI_img.png +val/254520_ALI_img.png +val/254524_ALI_img.png +val/254528_ALI_img.png +val/254529_ALI_img.png +val/254530_ALI_img.png +val/254532_ALI_img.png +val/254535_ALI_img.png +val/254539_ALI_img.png +val/254540_ALI_img.png +val/254541_ALI_img.png +val/254542_ALI_img.png +val/254550_ALI_img.png +val/254553_ALI_img.png +val/254561_ALI_img.png +val/254566_ALI_img.png +val/254570_ALI_img.png +val/254574_ALI_img.png +val/254577_ALI_img.png +val/254580_ALI_img.png +val/254584_ALI_img.png +val/254589_ALI_img.png +val/254597_ALI_img.png +val/254608_ALI_img.png +val/254610_ALI_img.png +val/254617_ALI_img.png +val/254621_ALI_img.png +val/254624_ALI_img.png +val/254625_ALI_img.png +val/254626_ALI_img.png +val/254634_ALI_img.png +val/254648_ALI_img.png +val/254653_ALI_img.png +val/254664_ALI_img.png +val/254666_ALI_img.png +val/254667_ALI_img.png +val/254669_ALI_img.png +val/254670_ALI_img.png +val/254673_DT_img.png +val/254674_ALI_img.png +val/254677_ALI_img.png +val/254685_ALI_img.png +val/254687_ALI_img.png +val/254688_ALI_img.png +val/254689_ALI_img.png +val/254693_ALI_img.png +val/254696_ALI_img.png +val/254700_ALI_img.png +val/254704_ALI_img.png +val/254707_ALI_img.png +val/254714_ALI_img.png +val/254718_ALI_img.png +val/254724_ALI_img.png +val/254725_ALI_img.png +val/254729_ALI_img.png +val/254731_ALI_img.png +val/254732_ALI_img.png +val/254735_ALI_img.png +val/254737_ALI_img.png +val/254749_ALI_img.png +val/254750_ALI_img.png +val/254751_ALI_img.png +val/254757_ALI_img.png +val/254766_ALI_img.png +val/254768_ALI_img.png +val/254769_ALI_img.png +val/254775_ALI_img.png +val/254776_ALI_img.png +val/254791_ALI_img.png +val/254792_ALI_img.png +val/254799_ALI_img.png +val/254801_ALI_img.png +val/254802_ALI_img.png +val/254806_ALI_img.png +val/254808_ALI_img.png +val/254814_ALI_img.png +val/254815_ALI_img.png +val/254822_ALI_img.png +val/254826_ALI_img.png +val/254833_ALI_img.png +val/254840_ALI_img.png +val/254844_ALI_img.png +val/254864_ALI_img.png +val/254866_ALI_img.png +val/254868_ALI_img.png +val/254872_ALI_img.png +val/254874_ALI_img.png +val/254878_ALI_img.png +val/254882_ALI_img.png +val/254884_ALI_img.png +val/254892_ALI_img.png +val/254898_ALI_img.png +val/254900_ALI_img.png +val/254909_ALI_img.png +val/254911_ALI_img.png +val/254912_ALI_img.png +val/254920_ALI_img.png +val/254921_ALI_img.png +val/254923_ALI_img.png +val/254928_ALI_img.png +val/254932_ALI_img.png +val/254935_ALI_img.png +val/254936_ALI_img.png +val/254938_ALI_img.png +val/254939_ALI_img.png +val/254945_ALI_img.png +val/254948_ALI_img.png +val/254950_ALI_img.png +val/254951_ALI_img.png +val/254955_ALI_img.png +val/254956_ALI_img.png +val/254959_ALI_img.png +val/254960_ALI_img.png +val/254962_ALI_img.png +val/254963_ALI_img.png +val/254964_ALI_img.png +val/254972_DT_img.png +val/254979_ALI_img.png +val/254985_ALI_img.png +val/254987_ALI_img.png +val/254990_ALI_img.png +val/254993_ALI_img.png +val/254994_ALI_img.png +val/254996_ALI_img.png +val/255002_ALI_img.png +val/255003_ALI_img.png +val/255004_ALI_img.png +val/255007_ALI_img.png +val/255013_ALI_img.png +val/255016_ALI_img.png +val/255022_ALI_img.png +val/255024_DT_img.png +val/255027_ALI_img.png +val/255032_ALI_img.png +val/255033_ALI_img.png +val/255037_ALI_img.png +val/255040_ALI_img.png +val/255044_ALI_img.png +val/255048_ALI_img.png +val/255049_ALI_img.png +val/255054_ALI_img.png +val/255058_ALI_img.png +val/255059_ALI_img.png +val/255068_ALI_img.png +val/255070_ALI_img.png +val/255073_ALI_img.png +val/255076_ALI_img.png +val/255089_ALI_img.png +val/255093_ALI_img.png +val/255095_ALI_img.png +val/255097_ALI_img.png +val/255099_ALI_img.png +val/255105_ALI_img.png +val/255114_ALI_img.png +val/255122_ALI_img.png +val/255123_ALI_img.png +val/255124_ALI_img.png +val/255125_ALI_img.png +val/255130_ALI_img.png +val/255134_ALI_img.png +val/255137_ALI_img.png +val/255143_ALI_img.png +val/255144_ALI_img.png +val/255148_ALI_img.png +val/255150_ALI_img.png +val/255152_ALI_img.png +val/255156_ALI_img.png +val/255157_ALI_img.png +val/255160_ALI_img.png +val/255165_ALI_img.png +val/255171_ALI_img.png +val/255172_ALI_img.png +val/255176_ALI_img.png +val/255179_ALI_img.png +val/255187_ALI_img.png +val/255189_ALI_img.png +val/255196_ALI_img.png +val/255200_ALI_img.png +val/255204_ALI_img.png +val/255206_ALI_img.png +val/255207_ALI_img.png +val/255213_ALI_img.png +val/255214_ALI_img.png +val/255221_ALI_img.png +val/255224_ALI_img.png +val/255229_ALI_img.png +val/255230_ALI_img.png +val/255237_ALI_img.png +val/255239_ALI_img.png +val/255241_ALI_img.png +val/255246_ALI_img.png +val/255247_ALI_img.png +val/255248_ALI_img.png +val/255255_ALI_img.png +val/255261_ALI_img.png +val/255262_ALI_img.png +val/255264_ALI_img.png +val/255267_ALI_img.png +val/255272_ALI_img.png +val/255274_ALI_img.png +val/255275_ALI_img.png +val/255276_ALI_img.png +val/255282_ALI_img.png +val/255287_ALI_img.png +val/255292_ALI_img.png +val/255296_ALI_img.png +val/255299_ALI_img.png +val/255307_ALI_img.png +val/255308_ALI_img.png +val/255313_ALI_img.png +val/255317_ALI_img.png +val/255322_ALI_img.png +val/255325_ALI_img.png +val/255328_ALI_img.png +val/255331_ALI_img.png +val/255340_ALI_img.png +val/255341_ALI_img.png +val/255345_ALI_img.png +val/255350_ALI_img.png +val/255353_ALI_img.png +val/255354_ALI_img.png +val/255359_ALI_img.png +val/255364_ALI_img.png +val/255370_ALI_img.png +val/255371_ALI_img.png +val/255378_ALI_img.png +val/255379_ALI_img.png +val/255382_ALI_img.png +val/255383_ALI_img.png +val/255388_ALI_img.png +val/255396_ALI_img.png +val/255397_ALI_img.png +val/255400_ALI_img.png +val/255402_ALI_img.png +val/255407_ALI_img.png +val/255416_ALI_img.png +val/255418_ALI_img.png +val/255419_ALI_img.png +val/255425_ALI_img.png +val/255428_ALI_img.png +val/255435_ALI_img.png +val/255438_ALI_img.png +val/255441_ALI_img.png +val/255443_ALI_img.png +val/255449_ALI_img.png +val/255450_ALI_img.png +val/255451_ALI_img.png +val/255452_ALI_img.png +val/255454_ALI_img.png +val/255455_ALI_img.png +val/255457_ALI_img.png +val/255465_ALI_img.png +val/255468_ALI_img.png +val/255472_ALI_img.png +val/255478_ALI_img.png +val/255482_ALI_img.png +val/255484_ALI_img.png +val/255487_ALI_img.png +val/255489_ALI_img.png +val/255490_ALI_img.png +val/255495_ALI_img.png +val/255500_ALI_img.png +val/255502_ALI_img.png +val/255503_ALI_img.png +val/255505_ALI_img.png +val/255515_ALI_img.png +val/255516_ALI_img.png +val/255533_ALI_img.png +val/255544_ALI_img.png +val/255552_ALI_img.png +val/255564_ALI_img.png +val/255568_ALI_img.png +val/255570_ALI_img.png +val/255571_ALI_img.png +val/255575_ALI_img.png +val/255582_ALI_img.png +val/255587_ALI_img.png +val/255591_ALI_img.png +val/255598_ALI_img.png +val/255601_ALI_img.png +val/255602_ALI_img.png +val/255610_ALI_img.png +val/255615_ALI_img.png +val/255622_ALI_img.png +val/255635_ALI_img.png +val/255638_DT_img.png +val/255640_ALI_img.png +val/255641_ALI_img.png +val/255643_ALI_img.png +val/255644_ALI_img.png +val/255645_ALI_img.png +val/255652_ALI_img.png +val/255654_ALI_img.png +val/255658_ALI_img.png +val/255659_ALI_img.png +val/255664_ALI_img.png +val/255665_ALI_img.png +val/255676_ALI_img.png +val/255683_ALI_img.png +val/255686_ALI_img.png +val/255687_ALI_img.png +val/255692_ALI_img.png +val/255697_ALI_img.png +val/255703_ALI_img.png +val/255716_ALI_img.png +val/255718_ALI_img.png +val/255732_ALI_img.png +val/255738_ALI_img.png +val/255739_ALI_img.png +val/255746_ALI_img.png +val/255756_ALI_img.png +val/255764_ALI_img.png +val/255766_ALI_img.png +val/255767_ALI_img.png +val/255768_ALI_img.png +val/255781_ALI_img.png +val/255786_ALI_img.png +val/255787_ALI_img.png +val/255796_ALI_img.png +val/255798_ALI_img.png +val/255801_ALI_img.png +val/255803_ALI_img.png +val/255806_ALI_img.png +val/255812_ALI_img.png +val/255817_ALI_img.png +val/255818_ALI_img.png +val/255827_ALI_img.png +val/255830_ALI_img.png +val/255833_ALI_img.png +val/255834_ALI_img.png +val/255838_ALI_img.png +val/255840_ALI_img.png +val/255851_ALI_img.png +val/255854_ALI_img.png +val/255855_ALI_img.png +val/255859_ALI_img.png +val/255863_ALI_img.png +val/255879_ALI_img.png +val/255880_ALI_img.png +val/255884_ALI_img.png +val/255893_ALI_img.png +val/255896_ALI_img.png +val/255904_ALI_img.png +val/255906_ALI_img.png +val/255909_ALI_img.png +val/255912_ALI_img.png +val/255915_ALI_img.png +val/255922_ALI_img.png +val/255933_ALI_img.png +val/255944_ALI_img.png +val/255946_ALI_img.png +val/255950_ALI_img.png +val/255951_ALI_img.png +val/255952_ALI_img.png +val/255953_ALI_img.png +val/255957_ALI_img.png +val/255959_ALI_img.png +val/255962_ALI_img.png +val/255967_ALI_img.png +val/255971_ALI_img.png +val/255974_ALI_img.png +val/255977_ALI_img.png +val/255980_ALI_img.png +val/255982_ALI_img.png +val/255984_ALI_img.png +val/255988_ALI_img.png +val/255989_ALI_img.png +val/255990_ALI_img.png +val/255993_ALI_img.png +val/255994_ALI_img.png +val/256000_ALI_img.png +val/256004_ALI_img.png +val/256005_ALI_img.png +val/256006_ALI_img.png +val/256008_ALI_img.png +val/256010_ALI_img.png +val/256022_ALI_img.png +val/256024_ALI_img.png +val/256027_ALI_img.png +val/256028_ALI_img.png +val/256030_ALI_img.png +val/256031_ALI_img.png +val/256043_ALI_img.png +val/256046_ALI_img.png +val/256055_ALI_img.png +val/256063_ALI_img.png +val/256064_ALI_img.png +val/256065_ALI_img.png +val/256069_ALI_img.png +val/256074_ALI_img.png +val/256075_ALI_img.png +val/256080_ALI_img.png +val/256081_ALI_img.png +val/256086_ALI_img.png +val/256095_ALI_img.png +val/256113_ALI_img.png +val/256119_ALI_img.png +val/256121_ALI_img.png +val/256124_DT_img.png +val/256134_ALI_img.png +val/256135_ALI_img.png +val/256144_ALI_img.png +val/256157_ALI_img.png +val/256161_ALI_img.png +val/256162_ALI_img.png +val/256163_ALI_img.png +val/256171_ALI_img.png +val/256178_ALI_img.png +val/256183_ALI_img.png +val/256186_ALI_img.png +val/256188_ALI_img.png +val/256193_ALI_img.png +val/256198_ALI_img.png +val/256201_ALI_img.png +val/256210_ALI_img.png +val/256214_ALI_img.png +val/256217_ALI_img.png +val/256218_ALI_img.png +val/256222_ALI_img.png +val/256224_ALI_img.png +val/256225_ALI_img.png +val/256227_ALI_img.png +val/256238_ALI_img.png +val/256245_ALI_img.png +val/256248_ALI_img.png +val/256253_ALI_img.png +val/256255_ALI_img.png +val/256260_ALI_img.png +val/256263_ALI_img.png +val/256266_ALI_img.png +val/256267_ALI_img.png +val/256270_ALI_img.png +val/256275_ALI_img.png +val/256277_ALI_img.png +val/256278_ALI_img.png +val/256286_ALI_img.png +val/256289_ALI_img.png +val/256290_ALI_img.png +val/256301_ALI_img.png +val/256315_ALI_img.png +val/256316_ALI_img.png +val/256322_ALI_img.png +val/256324_ALI_img.png +val/256325_ALI_img.png +val/256326_ALI_img.png +val/256336_ALI_img.png +val/256340_ALI_img.png +val/256342_ALI_img.png +val/256344_ALI_img.png +val/256346_ALI_img.png +val/256347_ALI_img.png +val/256351_ALI_img.png +val/256357_ALI_img.png +val/256359_ALI_img.png +val/256375_ALI_img.png +val/256377_ALI_img.png +val/256379_ALI_img.png +val/256381_ALI_img.png +val/256386_ALI_img.png +val/256387_ALI_img.png +val/256398_ALI_img.png +val/256404_ALI_img.png +val/256411_ALI_img.png +val/256420_ALI_img.png +val/256421_ALI_img.png +val/256424_ALI_img.png +val/256426_ALI_img.png +val/256428_ALI_img.png +val/256430_ALI_img.png +val/256432_ALI_img.png +val/256433_ALI_img.png +val/256440_ALI_img.png +val/256442_ALI_img.png +val/256446_ALI_img.png +val/256447_ALI_img.png +val/256450_ALI_img.png +val/256455_ALI_img.png +val/256456_ALI_img.png +val/256460_ALI_img.png +val/256463_ALI_img.png +val/256476_ALI_img.png +val/256487_ALI_img.png +val/256492_ALI_img.png +val/256496_ALI_img.png +val/256502_ALI_img.png +val/256503_ALI_img.png +val/256507_ALI_img.png +val/256511_ALI_img.png +val/256520_ALI_img.png +val/256521_ALI_img.png +val/256528_ALI_img.png +val/256529_ALI_img.png +val/256531_DT_img.png +val/256532_ALI_img.png +val/256533_ALI_img.png +val/256538_ALI_img.png +val/256541_ALI_img.png +val/256546_ALI_img.png +val/256547_ALI_img.png +val/256548_ALI_img.png +val/256555_ALI_img.png +val/256559_ALI_img.png +val/256560_ALI_img.png +val/256561_ALI_img.png +val/256563_ALI_img.png +val/256567_ALI_img.png +val/256571_ALI_img.png +val/256572_ALI_img.png +val/256578_ALI_img.png +val/256585_ALI_img.png +val/256588_ALI_img.png +val/256589_ALI_img.png +val/256590_ALI_img.png +val/256593_ALI_img.png +val/256595_ALI_img.png +val/256600_ALI_img.png +val/256601_ALI_img.png +val/256604_ALI_img.png +val/256605_ALI_img.png +val/256609_ALI_img.png +val/256610_ALI_img.png +val/256611_ALI_img.png +val/256614_ALI_img.png +val/256618_ALI_img.png +val/256619_ALI_img.png +val/256621_ALI_img.png +val/256630_ALI_img.png +val/256638_ALI_img.png +val/256639_ALI_img.png +val/256641_ALI_img.png +val/256642_ALI_img.png +val/256645_ALI_img.png +val/256647_ALI_img.png +val/256650_ALI_img.png +val/256660_ALI_img.png +val/256661_ALI_img.png +val/256663_ALI_img.png +val/256664_ALI_img.png +val/256665_ALI_img.png +val/256678_ALI_img.png +val/256690_ALI_img.png +val/256696_ALI_img.png +val/256698_ALI_img.png +val/256703_ALI_img.png +val/256707_ALI_img.png +val/256721_ALI_img.png +val/256724_ALI_img.png +val/256726_ALI_img.png +val/256731_ALI_img.png +val/256732_ALI_img.png +val/256733_ALI_img.png +val/256736_ALI_img.png +val/256740_ALI_img.png +val/256748_ALI_img.png +val/256755_ALI_img.png +val/256765_ALI_img.png +val/256767_ALI_img.png +val/256769_ALI_img.png +val/256770_ALI_img.png +val/256771_ALI_img.png +val/256772_ALI_img.png +val/256774_ALI_img.png +val/256785_ALI_img.png +val/256787_DT_img.png +val/256807_ALI_img.png +val/256815_ALI_img.png +val/256819_ALI_img.png +val/256822_ALI_img.png +val/256824_ALI_img.png +val/256827_ALI_img.png +val/256830_ALI_img.png +val/256835_ALI_img.png +val/256836_ALI_img.png +val/256843_ALI_img.png +val/256847_ALI_img.png +val/256848_ALI_img.png +val/256851_ALI_img.png +val/256856_ALI_img.png +val/256862_ALI_img.png +val/256866_ALI_img.png +val/256875_ALI_img.png +val/256887_ALI_img.png +val/256893_ALI_img.png +val/256895_ALI_img.png +val/256896_ALI_img.png +val/256900_ALI_img.png +val/256901_ALI_img.png +val/256904_ALI_img.png +val/256910_ALI_img.png +val/256912_ALI_img.png +val/256913_ALI_img.png +val/256915_ALI_img.png +val/256919_ALI_img.png +val/256930_ALI_img.png +val/256931_ALI_img.png +val/256932_ALI_img.png +val/256937_ALI_img.png +val/256939_ALI_img.png +val/256942_ALI_img.png +val/256945_ALI_img.png +val/256951_ALI_img.png +val/256954_ALI_img.png +val/256962_ALI_img.png +val/256965_ALI_img.png +val/256968_ALI_img.png +val/256970_ALI_img.png +val/256973_ALI_img.png +val/256974_ALI_img.png +val/256982_ALI_img.png +val/256986_ALI_img.png +val/256989_ALI_img.png +val/256991_ALI_img.png +val/256992_ALI_img.png +val/256993_ALI_img.png +val/256995_ALI_img.png +val/257002_ALI_img.png +val/257006_ALI_img.png +val/257010_ALI_img.png +val/257011_ALI_img.png +val/257019_ALI_img.png +val/257024_ALI_img.png +val/257033_ALI_img.png +val/257034_ALI_img.png +val/257036_ALI_img.png +val/257038_ALI_img.png +val/257044_ALI_img.png +val/257046_ALI_img.png +val/257047_ALI_img.png +val/257053_ALI_img.png +val/257054_ALI_img.png +val/257061_ALI_img.png +val/257068_ALI_img.png +val/257071_ALI_img.png +val/257072_ALI_img.png +val/257075_ALI_img.png +val/257077_ALI_img.png +val/257079_ALI_img.png +val/257087_ALI_img.png +val/257091_ALI_img.png +val/257092_ALI_img.png +val/257096_ALI_img.png +val/257098_ALI_img.png +val/257103_ALI_img.png +val/257107_ALI_img.png +val/257110_ALI_img.png +val/257117_ALI_img.png +val/257122_ALI_img.png +val/257125_ALI_img.png +val/257126_ALI_img.png +val/257128_ALI_img.png +val/257130_ALI_img.png +val/257133_ALI_img.png +val/257136_ALI_img.png +val/257137_ALI_img.png +val/257138_ALI_img.png +val/257139_ALI_img.png +val/257156_ALI_img.png +val/257160_ALI_img.png +val/257173_ALI_img.png +val/257174_ALI_img.png +val/257175_ALI_img.png +val/257177_ALI_img.png +val/257180_ALI_img.png +val/257181_ALI_img.png +val/257183_ALI_img.png +val/257186_ALI_img.png +val/257190_ALI_img.png +val/257191_ALI_img.png +val/257194_ALI_img.png +val/257198_ALI_img.png +val/257199_ALI_img.png +val/257202_ALI_img.png +val/257206_ALI_img.png +val/257209_ALI_img.png +val/257214_ALI_img.png +val/257218_ALI_img.png +val/257221_ALI_img.png +val/257226_ALI_img.png +val/257230_ALI_img.png +val/257233_ALI_img.png +val/257237_ALI_img.png +val/257238_ALI_img.png +val/257239_ALI_img.png +val/257241_ALI_img.png +val/257249_ALI_img.png +val/257256_ALI_img.png +val/257262_ALI_img.png +val/257264_ALI_img.png +val/257270_ALI_img.png +val/257271_ALI_img.png +val/257278_ALI_img.png +val/257280_ALI_img.png +val/257285_ALI_img.png +val/257289_ALI_img.png +val/257292_ALI_img.png +val/257299_ALI_img.png +val/257310_ALI_img.png +val/257316_ALI_img.png +val/257317_ALI_img.png +val/257322_ALI_img.png +val/257323_ALI_img.png +val/257326_ALI_img.png +val/257327_ALI_img.png +val/257330_ALI_img.png +val/257335_ALI_img.png +val/257337_ALI_img.png +val/257353_ALI_img.png +val/257358_ALI_img.png +val/257360_ALI_img.png +val/257361_ALI_img.png +val/257369_ALI_img.png +val/257370_ALI_img.png +val/257371_ALI_img.png +val/257376_ALI_img.png +val/257383_ALI_img.png +val/257384_ALI_img.png +val/257386_ALI_img.png +val/257390_ALI_img.png +val/257392_ALI_img.png +val/257397_ALI_img.png +val/257398_ALI_img.png +val/257399_ALI_img.png +val/257401_ALI_img.png +val/257405_ALI_img.png +val/257411_ALI_img.png +val/257413_ALI_img.png +val/257416_ALI_img.png +val/257419_ALI_img.png +val/257421_ALI_img.png +val/257423_ALI_img.png +val/257424_ALI_img.png +val/257425_ALI_img.png +val/257430_ALI_img.png +val/257433_ALI_img.png +val/257435_ALI_img.png +val/257436_ALI_img.png +val/257439_ALI_img.png +val/257448_ALI_img.png +val/257459_ALI_img.png +val/257461_ALI_img.png +val/257468_ALI_img.png +val/257471_ALI_img.png +val/257473_ALI_img.png +val/257475_ALI_img.png +val/257485_ALI_img.png +val/257488_ALI_img.png +val/257489_ALI_img.png +val/257490_ALI_img.png +val/257494_ALI_img.png +val/257496_ALI_img.png +val/257499_ALI_img.png +val/257535_ALI_img.png +val/257544_ALI_img.png +val/257548_ALI_img.png +val/257553_ALI_img.png +val/257555_ALI_img.png +val/257556_ALI_img.png +val/257561_ALI_img.png +val/257568_ALI_img.png +val/257569_ALI_img.png +val/257573_ALI_img.png +val/257580_ALI_img.png +val/257585_ALI_img.png +val/257586_ALI_img.png +val/257587_ALI_img.png +val/257589_ALI_img.png +val/257600_ALI_img.png +val/257602_ALI_img.png +val/257607_ALI_img.png +val/257608_ALI_img.png +val/257611_ALI_img.png +val/257616_ALI_img.png +val/257624_ALI_img.png +val/257629_ALI_img.png +val/257636_ALI_img.png +val/257640_ALI_img.png +val/257642_ALI_img.png +val/257645_ALI_img.png +val/257651_ALI_img.png +val/257654_ALI_img.png +val/257655_ALI_img.png +val/257656_ALI_img.png +val/257663_ALI_img.png +val/257664_ALI_img.png +val/257665_ALI_img.png +val/257670_ALI_img.png +val/257680_ALI_img.png +val/257685_ALI_img.png +val/257695_ALI_img.png +val/257702_ALI_img.png +val/257703_ALI_img.png +val/257704_ALI_img.png +val/257712_ALI_img.png +val/257713_ALI_img.png +val/257716_ALI_img.png +val/257719_ALI_img.png +val/257722_ALI_img.png +val/257723_ALI_img.png +val/257725_ALI_img.png +val/257741_ALI_img.png +val/257746_ALI_img.png +val/257747_ALI_img.png +val/257761_ALI_img.png +val/257764_ALI_img.png +val/257765_ALI_img.png +val/257768_ALI_img.png +val/257773_ALI_img.png +val/257776_ALI_img.png +val/257778_ALI_img.png +val/257779_ALI_img.png +val/257781_ALI_img.png +val/257785_ALI_img.png +val/257787_ALI_img.png +val/257789_ALI_img.png +val/257792_ALI_img.png +val/257797_ALI_img.png +val/257807_ALI_img.png +val/257811_ALI_img.png +val/257813_ALI_img.png +val/257814_ALI_img.png +val/257816_ALI_img.png +val/257817_ALI_img.png +val/257818_ALI_img.png +val/257820_ALI_img.png +val/257823_ALI_img.png +val/257824_ALI_img.png +val/257828_ALI_img.png +val/257829_ALI_img.png +val/257830_ALI_img.png +val/257832_ALI_img.png +val/257840_ALI_img.png +val/257841_ALI_img.png +val/257846_ALI_img.png +val/257850_ALI_img.png +val/257858_ALI_img.png +val/257862_ALI_img.png +val/257864_ALI_img.png +val/257866_ALI_img.png +val/257869_ALI_img.png +val/257870_ALI_img.png +val/257880_ALI_img.png +val/257883_ALI_img.png +val/257886_ALI_img.png +val/257887_ALI_img.png +val/257893_ALI_img.png +val/257896_ALI_img.png +val/257899_ALI_img.png +val/257901_ALI_img.png +val/257904_ALI_img.png +val/257910_ALI_img.png +val/257914_ALI_img.png +val/257915_ALI_img.png +val/257917_ALI_img.png +val/257928_ALI_img.png +val/257933_ALI_img.png +val/257945_ALI_img.png +val/257950_ALI_img.png +val/257954_ALI_img.png +val/257955_ALI_img.png +val/257956_ALI_img.png +val/257957_ALI_img.png +val/257961_ALI_img.png +val/257963_ALI_img.png +val/257967_ALI_img.png +val/257977_ALI_img.png +val/257984_ALI_img.png +val/257985_ALI_img.png +val/257986_ALI_img.png +val/257990_ALI_img.png +val/257993_ALI_img.png +val/257998_ALI_img.png +val/258000_ALI_img.png +val/258006_ALI_img.png +val/258010_ALI_img.png +val/258014_ALI_img.png +val/258019_ALI_img.png +val/258027_ALI_img.png +val/258030_ALI_img.png +val/258033_ALI_img.png +val/258036_ALI_img.png +val/258038_ALI_img.png +val/258049_ALI_img.png +val/258056_ALI_img.png +val/258057_ALI_img.png +val/258062_ALI_img.png +val/258066_ALI_img.png +val/258067_ALI_img.png +val/258068_ALI_img.png +val/258071_ALI_img.png +val/258076_ALI_img.png +val/258080_ALI_img.png +val/258081_ALI_img.png +val/258085_ALI_img.png +val/258089_ALI_img.png +val/258098_ALI_img.png +val/258100_ALI_img.png +val/258101_ALI_img.png +val/258104_ALI_img.png +val/258106_ALI_img.png +val/258111_ALI_img.png +val/258112_ALI_img.png +val/258115_ALI_img.png +val/258116_ALI_img.png +val/258129_ALI_img.png +val/258136_ALI_img.png +val/258137_ALI_img.png +val/258138_ALI_img.png +val/258139_ALI_img.png +val/258142_ALI_img.png +val/258148_ALI_img.png +val/258150_ALI_img.png +val/258151_ALI_img.png +val/258153_ALI_img.png +val/258155_ALI_img.png +val/258156_ALI_img.png +val/258157_ALI_img.png +val/258158_ALI_img.png +val/258159_ALI_img.png +val/258168_ALI_img.png +val/258170_ALI_img.png +val/258178_ALI_img.png +val/258187_ALI_img.png +val/258192_ALI_img.png +val/258199_ALI_img.png +val/258202_ALI_img.png +val/258206_ALI_img.png +val/258208_ALI_img.png +val/258210_ALI_img.png +val/258213_ALI_img.png +val/258214_ALI_img.png +val/258226_ALI_img.png +val/258230_ALI_img.png +val/258233_ALI_img.png +val/258234_ALI_img.png +val/258237_ALI_img.png +val/258243_ALI_img.png +val/258245_ALI_img.png +val/258247_ALI_img.png +val/258251_ALI_img.png +val/258253_ALI_img.png +val/258255_ALI_img.png +val/258260_ALI_img.png +val/258261_ALI_img.png +val/258263_ALI_img.png +val/258272_ALI_img.png +val/258277_ALI_img.png +val/258281_ALI_img.png +val/258282_ALI_img.png +val/258287_ALI_img.png +val/258290_ALI_img.png +val/258291_ALI_img.png +val/258303_ALI_img.png +val/258307_ALI_img.png +val/258308_ALI_img.png +val/258320_ALI_img.png +val/258321_ALI_img.png +val/258323_ALI_img.png +val/258330_ALI_img.png +val/258339_ALI_img.png +val/258346_ALI_img.png +val/258347_ALI_img.png +val/258348_ALI_img.png +val/258349_ALI_img.png +val/258350_ALI_img.png +val/258363_ALI_img.png +val/258366_ALI_img.png +val/258371_ALI_img.png +val/258375_ALI_img.png +val/258376_ALI_img.png +val/258380_ALI_img.png +val/258386_ALI_img.png +val/258387_ALI_img.png +val/258392_ALI_img.png +val/258395_ALI_img.png +val/258398_ALI_img.png +val/258402_ALI_img.png +val/258404_ALI_img.png +val/258410_ALI_img.png +val/258417_ALI_img.png +val/258420_ALI_img.png +val/258424_ALI_img.png +val/258426_ALI_img.png +val/258437_ALI_img.png +val/258438_ALI_img.png +val/258440_ALI_img.png +val/258445_ALI_img.png +val/258447_ALI_img.png +val/258453_ALI_img.png +val/258454_ALI_img.png +val/258457_ALI_img.png +val/258465_ALI_img.png +val/258466_ALI_img.png +val/258474_ALI_img.png +val/258475_ALI_img.png +val/258476_ALI_img.png +val/258484_ALI_img.png +val/258488_ALI_img.png +val/258490_ALI_img.png +val/258493_ALI_img.png +val/258495_ALI_img.png +val/258501_ALI_img.png +val/258506_ALI_img.png +val/258507_ALI_img.png +val/258508_ALI_img.png +val/258513_ALI_img.png +val/258519_ALI_img.png +val/258521_ALI_img.png +val/258522_ALI_img.png +val/258527_ALI_img.png +val/258533_ALI_img.png +val/258536_ALI_img.png +val/258537_ALI_img.png +val/258548_ALI_img.png +val/258554_ALI_img.png +val/258555_ALI_img.png +val/258560_ALI_img.png +val/258569_ALI_img.png +val/258577_ALI_img.png +val/258581_ALI_img.png +val/258583_ALI_img.png +val/258593_ALI_img.png +val/258609_ALI_img.png +val/258610_ALI_img.png +val/258613_ALI_img.png +val/258622_ALI_img.png +val/258634_ALI_img.png +val/258641_ALI_img.png +val/258647_ALI_img.png +val/258656_ALI_img.png +val/258657_ALI_img.png +val/258659_ALI_img.png +val/258663_ALI_img.png +val/258666_ALI_img.png +val/258668_ALI_img.png +val/258669_ALI_img.png +val/258673_ALI_img.png +val/258678_ALI_img.png +val/258680_ALI_img.png +val/258682_ALI_img.png +val/258684_ALI_img.png +val/258690_ALI_img.png +val/258696_ALI_img.png +val/258701_ALI_img.png +val/258702_ALI_img.png +val/258703_ALI_img.png +val/258704_ALI_img.png +val/258711_ALI_img.png +val/258714_ALI_img.png +val/258716_ALI_img.png +val/258720_ALI_img.png +val/258732_ALI_img.png +val/258733_ALI_img.png +val/258739_ALI_img.png +val/258746_ALI_img.png +val/258751_ALI_img.png +val/258752_ALI_img.png +val/258760_ALI_img.png +val/258767_ALI_img.png +val/258769_ALI_img.png +val/258775_ALI_img.png +val/258786_ALI_img.png +val/258787_ALI_img.png +val/258790_ALI_img.png +val/258792_ALI_img.png +val/258795_ALI_img.png +val/258800_ALI_img.png +val/258801_ALI_img.png +val/258802_ALI_img.png +val/258805_ALI_img.png +val/258809_ALI_img.png +val/258811_ALI_img.png +val/258812_ALI_img.png +val/258815_ALI_img.png +val/258828_ALI_img.png +val/258831_ALI_img.png +val/258832_ALI_img.png +val/258839_ALI_img.png +val/258848_ALI_img.png +val/258849_ALI_img.png +val/258850_ALI_img.png +val/258851_ALI_img.png +val/258853_ALI_img.png +val/258854_ALI_img.png +val/258857_ALI_img.png +val/258859_ALI_img.png +val/258861_ALI_img.png +val/258872_ALI_img.png +val/258878_ALI_img.png +val/258884_ALI_img.png +val/258886_ALI_img.png +val/258887_ALI_img.png +val/258897_ALI_img.png +val/258901_ALI_img.png +val/258906_ALI_img.png +val/258913_ALI_img.png +val/258916_ALI_img.png +val/258930_ALI_img.png +val/258933_ALI_img.png +val/258934_ALI_img.png +val/258935_ALI_img.png +val/258941_ALI_img.png +val/258942_ALI_img.png +val/258947_ALI_img.png +val/258953_ALI_img.png +val/258958_ALI_img.png +val/258962_ALI_img.png +val/258964_ALI_img.png +val/258969_ALI_img.png +val/258973_ALI_img.png +val/258975_ALI_img.png +val/258978_ALI_img.png +val/258980_ALI_img.png +val/258981_ALI_img.png +val/258982_ALI_img.png +val/258984_ALI_img.png +val/258992_ALI_img.png +val/258997_ALI_img.png +val/259000_ALI_img.png +val/259002_ALI_img.png +val/259005_ALI_img.png +val/259006_ALI_img.png +val/259009_ALI_img.png +val/259014_ALI_img.png +val/259015_ALI_img.png +val/259020_ALI_img.png +val/259023_ALI_img.png +val/259025_ALI_img.png +val/259036_ALI_img.png +val/259038_ALI_img.png +val/259040_ALI_img.png +val/259046_ALI_img.png +val/259053_ALI_img.png +val/259055_ALI_img.png +val/259067_ALI_img.png +val/259068_ALI_img.png +val/259075_ALI_img.png +val/259078_ALI_img.png +val/259081_ALI_img.png +val/259086_ALI_img.png +val/259092_ALI_img.png +val/259095_ALI_img.png +val/259097_ALI_img.png +val/259101_ALI_img.png +val/259102_ALI_img.png +val/259111_ALI_img.png +val/259112_ALI_img.png +val/259113_ALI_img.png +val/259124_DT_img.png +val/259127_ALI_img.png +val/259128_ALI_img.png +val/259129_ALI_img.png +val/259133_ALI_img.png +val/259134_ALI_img.png +val/259149_ALI_img.png +val/259159_ALI_img.png +val/259160_ALI_img.png +val/259166_ALI_img.png +val/259168_ALI_img.png +val/259169_ALI_img.png +val/259173_ALI_img.png +val/259174_ALI_img.png +val/259175_ALI_img.png +val/259177_ALI_img.png +val/259178_ALI_img.png +val/259183_ALI_img.png +val/259185_ALI_img.png +val/259186_ALI_img.png +val/259190_ALI_img.png +val/259194_ALI_img.png +val/259197_ALI_img.png +val/259201_ALI_img.png +val/259204_ALI_img.png +val/259210_ALI_img.png +val/259212_ALI_img.png +val/259220_ALI_img.png +val/259222_ALI_img.png +val/259225_ALI_img.png +val/259230_ALI_img.png +val/259231_ALI_img.png +val/259239_ALI_img.png +val/259240_ALI_img.png +val/259241_ALI_img.png +val/259243_ALI_img.png +val/259250_ALI_img.png +val/259263_ALI_img.png +val/259268_ALI_img.png +val/259271_ALI_img.png +val/259276_ALI_img.png +val/259277_ALI_img.png +val/259282_ALI_img.png +val/259292_ALI_img.png +val/259294_ALI_img.png +val/259297_ALI_img.png +val/259298_ALI_img.png +val/259299_ALI_img.png +val/259300_ALI_img.png +val/259301_ALI_img.png +val/259305_ALI_img.png +val/259309_ALI_img.png +val/259312_ALI_img.png +val/259313_ALI_img.png +val/259315_ALI_img.png +val/259318_ALI_img.png +val/259324_ALI_img.png +val/259328_ALI_img.png +val/259332_ALI_img.png +val/259335_ALI_img.png +val/259346_ALI_img.png +val/259351_ALI_img.png +val/259352_ALI_img.png +val/259354_ALI_img.png +val/259355_ALI_img.png +val/259357_ALI_img.png +val/259361_ALI_img.png +val/259364_ALI_img.png +val/259367_ALI_img.png +val/259371_ALI_img.png +val/259372_ALI_img.png +val/259377_ALI_img.png +val/259379_ALI_img.png +val/259383_ALI_img.png +val/259384_ALI_img.png +val/259388_ALI_img.png +val/259392_ALI_img.png +val/259393_ALI_img.png +val/259394_ALI_img.png +val/259396_ALI_img.png +val/259397_ALI_img.png +val/259400_ALI_img.png +val/259401_ALI_img.png +val/259405_ALI_img.png +val/259411_ALI_img.png +val/259413_ALI_img.png +val/259414_ALI_img.png +val/259417_ALI_img.png +val/259419_ALI_img.png +val/259429_ALI_img.png +val/259436_ALI_img.png +val/259438_ALI_img.png +val/259440_ALI_img.png +val/259442_ALI_img.png +val/259444_ALI_img.png +val/259446_ALI_img.png +val/259450_ALI_img.png +val/259452_ALI_img.png +val/259453_ALI_img.png +val/259455_ALI_img.png +val/259456_ALI_img.png +val/259468_ALI_img.png +val/259471_ALI_img.png +val/259475_ALI_img.png +val/259477_ALI_img.png +val/259478_ALI_img.png +val/259482_ALI_img.png +val/259483_ALI_img.png +val/259486_ALI_img.png +val/259487_ALI_img.png +val/259488_ALI_img.png +val/259489_ALI_img.png +val/259492_DT_img.png +val/259496_ALI_img.png +val/259500_ALI_img.png +val/259501_ALI_img.png +val/259504_ALI_img.png +val/259508_ALI_img.png +val/259509_ALI_img.png +val/259511_ALI_img.png +val/259513_ALI_img.png +val/259514_ALI_img.png +val/259517_ALI_img.png +val/259523_ALI_img.png +val/259528_ALI_img.png +val/259532_ALI_img.png +val/259535_ALI_img.png +val/259537_ALI_img.png +val/259558_ALI_img.png +val/259567_ALI_img.png +val/259570_ALI_img.png +val/259571_ALI_img.png +val/259574_ALI_img.png +val/259575_ALI_img.png +val/259578_ALI_img.png +val/259581_ALI_img.png +val/259582_ALI_img.png +val/259588_ALI_img.png +val/259592_ALI_img.png +val/259593_ALI_img.png +val/259594_ALI_img.png +val/259597_ALI_img.png +val/259602_ALI_img.png +val/259610_ALI_img.png +val/259612_ALI_img.png +val/259616_ALI_img.png +val/259617_ALI_img.png +val/259619_ALI_img.png +val/259621_ALI_img.png +val/259623_ALI_img.png +val/259630_ALI_img.png +val/259640_ALI_img.png +val/259644_ALI_img.png +val/259650_ALI_img.png +val/259658_ALI_img.png +val/259669_ALI_img.png +val/259674_ALI_img.png +val/259681_ALI_img.png +val/259690_ALI_img.png +val/259693_ALI_img.png +val/259695_ALI_img.png +val/259697_ALI_img.png +val/259701_ALI_img.png +val/259703_ALI_img.png +val/259704_ALI_img.png +val/259705_ALI_img.png +val/259712_ALI_img.png +val/259718_ALI_img.png +val/259721_ALI_img.png +val/259727_ALI_img.png +val/259733_ALI_img.png +val/259735_ALI_img.png +val/259736_ALI_img.png +val/259738_ALI_img.png +val/259740_ALI_img.png +val/259743_ALI_img.png +val/259750_ALI_img.png +val/259757_ALI_img.png +val/259761_ALI_img.png +val/259765_ALI_img.png +val/259777_ALI_img.png +val/259778_ALI_img.png +val/259785_ALI_img.png +val/259789_ALI_img.png +val/259796_ALI_img.png +val/259800_ALI_img.png +val/259810_ALI_img.png +val/259818_ALI_img.png +val/259821_ALI_img.png +val/259825_ALI_img.png +val/259826_ALI_img.png +val/259831_ALI_img.png +val/259840_ALI_img.png +val/259852_ALI_img.png +val/259858_ALI_img.png +val/259861_ALI_img.png +val/259864_ALI_img.png +val/259873_ALI_img.png +val/259883_ALI_img.png +val/259895_ALI_img.png +val/259903_ALI_img.png +val/259904_ALI_img.png +val/259917_ALI_img.png +val/259919_ALI_img.png +val/259922_ALI_img.png +val/259925_ALI_img.png +val/259926_ALI_img.png +val/259929_ALI_img.png +val/259932_ALI_img.png +val/259933_ALI_img.png +val/259937_ALI_img.png +val/259948_ALI_img.png +val/259952_ALI_img.png +val/259960_ALI_img.png +val/259961_ALI_img.png +val/259963_ALI_img.png +val/259964_ALI_img.png +val/259967_ALI_img.png +val/259973_ALI_img.png +val/259975_ALI_img.png +val/259976_ALI_img.png +val/259978_ALI_img.png +val/259981_ALI_img.png +val/259982_ALI_img.png +val/259987_ALI_img.png +val/259990_ALI_img.png +val/259991_ALI_img.png +val/260000_ALI_img.png +val/260004_ALI_img.png +val/260007_ALI_img.png +val/260024_ALI_img.png +val/260025_ALI_img.png +val/260026_ALI_img.png +val/260028_ALI_img.png +val/260029_ALI_img.png +val/260033_ALI_img.png +val/260038_ALI_img.png +val/260040_ALI_img.png +val/260045_ALI_img.png +val/260046_ALI_img.png +val/260051_ALI_img.png +val/260052_ALI_img.png +val/260058_ALI_img.png +val/260059_ALI_img.png +val/260072_ALI_img.png +val/260074_ALI_img.png +val/260080_ALI_img.png +val/260082_ALI_img.png +val/260084_ALI_img.png +val/260087_ALI_img.png +val/260090_ALI_img.png +val/260100_ALI_img.png +val/260102_ALI_img.png +val/260103_ALI_img.png +val/260110_ALI_img.png +val/260114_ALI_img.png +val/260117_ALI_img.png +val/260124_ALI_img.png +val/260126_ALI_img.png +val/260127_ALI_img.png +val/260131_ALI_img.png +val/260137_ALI_img.png +val/260140_ALI_img.png +val/260144_ALI_img.png +val/260147_ALI_img.png +val/260151_ALI_img.png +val/260153_ALI_img.png +val/260157_ALI_img.png +val/260169_ALI_img.png +val/260176_ALI_img.png +val/260183_ALI_img.png +val/260193_ALI_img.png +val/260194_ALI_img.png +val/260199_ALI_img.png +val/260200_ALI_img.png +val/260208_ALI_img.png +val/260212_ALI_img.png +val/260216_ALI_img.png +val/260217_ALI_img.png +val/260218_ALI_img.png +val/260224_ALI_img.png +val/260231_ALI_img.png +val/260237_ALI_img.png +val/260242_ALI_img.png +val/260248_ALI_img.png +val/260250_ALI_img.png +val/260252_ALI_img.png +val/260255_ALI_img.png +val/260256_ALI_img.png +val/260261_ALI_img.png +val/260262_ALI_img.png +val/260282_ALI_img.png +val/260285_ALI_img.png +val/260286_ALI_img.png +val/260293_ALI_img.png +val/260295_ALI_img.png +val/260299_ALI_img.png +val/260300_ALI_img.png +val/260302_ALI_img.png +val/260303_ALI_img.png +val/260304_ALI_img.png +val/260305_ALI_img.png +val/260307_ALI_img.png +val/260309_ALI_img.png +val/260312_ALI_img.png +val/260315_ALI_img.png +val/260318_ALI_img.png +val/260323_ALI_img.png +val/260329_ALI_img.png +val/260333_ALI_img.png +val/260334_ALI_img.png +val/260338_ALI_img.png +val/260342_ALI_img.png +val/260347_ALI_img.png +val/260349_ALI_img.png +val/260352_ALI_img.png +val/260361_ALI_img.png +val/260366_ALI_img.png +val/260368_ALI_img.png +val/260370_ALI_img.png +val/260372_ALI_img.png +val/260374_ALI_img.png +val/260380_ALI_img.png +val/260385_ALI_img.png +val/260387_ALI_img.png +val/260388_ALI_img.png +val/260391_ALI_img.png +val/260397_ALI_img.png +val/260398_ALI_img.png +val/260399_ALI_img.png +val/260400_ALI_img.png +val/260402_ALI_img.png +val/260407_ALI_img.png +val/260414_ALI_img.png +val/260415_ALI_img.png +val/260417_ALI_img.png +val/260423_ALI_img.png +val/260427_ALI_img.png +val/260431_ALI_img.png +val/260440_ALI_img.png +val/260444_ALI_img.png +val/260446_ALI_img.png +val/260447_ALI_img.png +val/260449_ALI_img.png +val/260451_ALI_img.png +val/260453_ALI_img.png +val/260455_ALI_img.png +val/260460_ALI_img.png +val/260472_ALI_img.png +val/260481_ALI_img.png +val/260486_ALI_img.png +val/260488_ALI_img.png +val/260496_ALI_img.png +val/260500_ALI_img.png +val/260501_ALI_img.png +val/260506_ALI_img.png +val/260509_ALI_img.png +val/260514_ALI_img.png +val/260518_ALI_img.png +val/260519_ALI_img.png +val/260523_ALI_img.png +val/260527_ALI_img.png +val/260534_ALI_img.png +val/260537_ALI_img.png +val/260538_ALI_img.png +val/260539_ALI_img.png +val/260540_ALI_img.png +val/260541_ALI_img.png +val/260547_ALI_img.png +val/260555_ALI_img.png +val/260556_ALI_img.png +val/260564_ALI_img.png +val/260567_ALI_img.png +val/260569_ALI_img.png +val/260573_ALI_img.png +val/260578_ALI_img.png +val/260579_ALI_img.png +val/260580_ALI_img.png +val/260582_ALI_img.png +val/260586_ALI_img.png +val/260589_ALI_img.png +val/260592_ALI_img.png +val/260597_ALI_img.png +val/260599_ALI_img.png +val/260601_ALI_img.png +val/260602_ALI_img.png +val/260607_ALI_img.png +val/260608_ALI_img.png +val/260612_ALI_img.png +val/260615_ALI_img.png +val/260617_ALI_img.png +val/260618_ALI_img.png +val/260636_ALI_img.png +val/260637_ALI_img.png +val/260643_ALI_img.png +val/260652_ALI_img.png +val/260655_ALI_img.png +val/260658_ALI_img.png +val/260660_ALI_img.png +val/260661_ALI_img.png +val/260662_ALI_img.png +val/260664_ALI_img.png +val/260679_ALI_img.png +val/260681_ALI_img.png +val/260685_ALI_img.png +val/260686_ALI_img.png +val/260692_ALI_img.png +val/260693_ALI_img.png +val/260696_ALI_img.png +val/260702_ALI_img.png +val/260709_ALI_img.png +val/260710_ALI_img.png +val/260711_ALI_img.png +val/260712_ALI_img.png +val/260713_ALI_img.png +val/260716_ALI_img.png +val/260722_ALI_img.png +val/260725_ALI_img.png +val/260729_ALI_img.png +val/260732_ALI_img.png +val/260733_ALI_img.png +val/260736_ALI_img.png +val/260739_ALI_img.png +val/260746_ALI_img.png +val/260748_ALI_img.png +val/260752_ALI_img.png +val/260754_ALI_img.png +val/260755_ALI_img.png +val/260767_ALI_img.png +val/260771_ALI_img.png +val/260787_ALI_img.png +val/260788_ALI_img.png +val/260794_ALI_img.png +val/260795_ALI_img.png +val/260802_ALI_img.png +val/260805_ALI_img.png +val/260809_ALI_img.png +val/260810_ALI_img.png +val/260814_ALI_img.png +val/260816_ALI_img.png +val/260822_ALI_img.png +val/260825_ALI_img.png +val/260829_ALI_img.png +val/260837_ALI_img.png +val/260841_ALI_img.png +val/260846_ALI_img.png +val/260850_ALI_img.png +val/260852_ALI_img.png +val/260855_ALI_img.png +val/260856_ALI_img.png +val/260859_ALI_img.png +val/260871_ALI_img.png +val/260877_ALI_img.png +val/260879_ALI_img.png +val/260887_ALI_img.png +val/260890_ALI_img.png +val/260901_ALI_img.png +val/260903_ALI_img.png +val/260908_ALI_img.png +val/260911_ALI_img.png +val/260912_ALI_img.png +val/260915_ALI_img.png +val/260918_ALI_img.png +val/260919_ALI_img.png +val/260923_ALI_img.png +val/260931_ALI_img.png +val/260932_ALI_img.png +val/260934_ALI_img.png +val/260939_ALI_img.png +val/260940_ALI_img.png +val/260945_ALI_img.png +val/260947_ALI_img.png +val/260950_ALI_img.png +val/260952_ALI_img.png +val/260971_ALI_img.png +val/260977_ALI_img.png +val/260983_ALI_img.png +val/260985_ALI_img.png +val/261003_ALI_img.png +val/261004_ALI_img.png +val/261005_ALI_img.png +val/261007_ALI_img.png +val/261012_ALI_img.png +val/261014_ALI_img.png +val/261023_ALI_img.png +val/261026_ALI_img.png +val/261031_ALI_img.png +val/261033_ALI_img.png +val/261034_ALI_img.png +val/261042_ALI_img.png +val/261045_ALI_img.png +val/261046_ALI_img.png +val/261049_ALI_img.png +val/261052_ALI_img.png +val/261053_ALI_img.png +val/261055_ALI_img.png +val/261063_ALI_img.png +val/261065_ALI_img.png +val/261070_ALI_img.png +val/261075_ALI_img.png +val/261078_ALI_img.png +val/261081_ALI_img.png +val/261092_ALI_img.png +val/261094_ALI_img.png +val/261095_ALI_img.png +val/261103_ALI_img.png +val/261113_ALI_img.png +val/261115_ALI_img.png +val/261117_ALI_img.png +val/261121_ALI_img.png +val/261125_ALI_img.png +val/261128_ALI_img.png +val/261136_ALI_img.png +val/261148_DT_img.png +val/261151_ALI_img.png +val/261152_ALI_img.png +val/261161_ALI_img.png +val/261164_ALI_img.png +val/261165_ALI_img.png +val/261173_ALI_img.png +val/261178_ALI_img.png +val/261181_ALI_img.png +val/261182_ALI_img.png +val/261185_ALI_img.png +val/261186_ALI_img.png +val/261189_ALI_img.png +val/261191_ALI_img.png +val/261195_ALI_img.png +val/261197_ALI_img.png +val/261198_ALI_img.png +val/261199_ALI_img.png +val/261207_ALI_img.png +val/261208_ALI_img.png +val/261212_ALI_img.png +val/261215_ALI_img.png +val/261224_ALI_img.png +val/261238_ALI_img.png +val/261249_ALI_img.png +val/261252_ALI_img.png +val/261254_ALI_img.png +val/261262_ALI_img.png +val/261266_ALI_img.png +val/261268_ALI_img.png +val/261272_ALI_img.png +val/261291_ALI_img.png +val/261292_ALI_img.png +val/261296_ALI_img.png +val/261301_ALI_img.png +val/261303_ALI_img.png +val/261312_ALI_img.png +val/261313_ALI_img.png +val/261315_ALI_img.png +val/261322_ALI_img.png +val/261327_ALI_img.png +val/261329_ALI_img.png +val/261338_ALI_img.png +val/261339_ALI_img.png +val/261341_ALI_img.png +val/261342_ALI_img.png +val/261347_ALI_img.png +val/261348_ALI_img.png +val/261354_ALI_img.png +val/261356_ALI_img.png +val/261358_ALI_img.png +val/261361_ALI_img.png +val/261363_ALI_img.png +val/261365_ALI_img.png +val/261367_ALI_img.png +val/261372_ALI_img.png +val/261374_ALI_img.png +val/261381_ALI_img.png +val/261382_ALI_img.png +val/261405_ALI_img.png +val/261409_ALI_img.png +val/261412_ALI_img.png +val/261419_ALI_img.png +val/261424_ALI_img.png +val/261435_ALI_img.png +val/261436_ALI_img.png +val/261440_ALI_img.png +val/261446_ALI_img.png +val/261449_ALI_img.png +val/261450_ALI_img.png +val/261463_ALI_img.png +val/261468_ALI_img.png +val/261470_ALI_img.png +val/261475_ALI_img.png +val/261478_ALI_img.png +val/261482_ALI_img.png +val/261486_ALI_img.png +val/261490_ALI_img.png +val/261492_ALI_img.png +val/261493_ALI_img.png +val/261498_ALI_img.png +val/261501_ALI_img.png +val/261502_ALI_img.png +val/261512_ALI_img.png +val/261518_ALI_img.png +val/261526_ALI_img.png +val/261533_ALI_img.png +val/261534_ALI_img.png +val/261535_ALI_img.png +val/261542_ALI_img.png +val/261550_ALI_img.png +val/261551_ALI_img.png +val/261552_ALI_img.png +val/261553_ALI_img.png +val/261560_ALI_img.png +val/261564_ALI_img.png +val/261566_ALI_img.png +val/261568_ALI_img.png +val/261575_ALI_img.png +val/261590_ALI_img.png +val/261594_ALI_img.png +val/261598_ALI_img.png +val/261599_ALI_img.png +val/261601_ALI_img.png +val/261603_ALI_img.png +val/261609_ALI_img.png +val/261611_ALI_img.png +val/261615_ALI_img.png +val/261622_ALI_img.png +val/261623_ALI_img.png +val/261627_ALI_img.png +val/261629_ALI_img.png +val/261632_ALI_img.png +val/261644_ALI_img.png +val/261647_ALI_img.png +val/261648_ALI_img.png +val/261658_ALI_img.png +val/261661_ALI_img.png +val/261667_ALI_img.png +val/261670_ALI_img.png +val/261678_ALI_img.png +val/261683_ALI_img.png +val/261689_ALI_img.png +val/261693_ALI_img.png +val/261696_ALI_img.png +val/261699_ALI_img.png +val/261700_ALI_img.png +val/261703_ALI_img.png +val/261709_ALI_img.png +val/261713_ALI_img.png +val/261720_ALI_img.png +val/261723_DT_img.png +val/261726_ALI_img.png +val/261727_ALI_img.png +val/261729_ALI_img.png +val/261733_ALI_img.png +val/261743_ALI_img.png +val/261748_ALI_img.png +val/261750_ALI_img.png +val/261751_ALI_img.png +val/261752_ALI_img.png +val/261754_ALI_img.png +val/261756_ALI_img.png +val/261762_ALI_img.png +val/261764_ALI_img.png +val/261780_ALI_img.png +val/261782_ALI_img.png +val/261784_ALI_img.png +val/261785_ALI_img.png +val/261795_ALI_img.png +val/261796_ALI_img.png +val/261798_ALI_img.png +val/261799_ALI_img.png +val/261802_ALI_img.png +val/261807_ALI_img.png +val/261808_ALI_img.png +val/261810_ALI_img.png +val/261812_ALI_img.png +val/261813_ALI_img.png +val/261818_ALI_img.png +val/261821_ALI_img.png +val/261825_ALI_img.png +val/261831_ALI_img.png +val/261833_ALI_img.png +val/261834_ALI_img.png +val/261842_ALI_img.png +val/261845_ALI_img.png +val/261849_ALI_img.png +val/261855_ALI_img.png +val/261856_ALI_img.png +val/261857_ALI_img.png +val/261870_ALI_img.png +val/261874_ALI_img.png +val/261879_ALI_img.png +val/261880_ALI_img.png +val/261884_ALI_img.png +val/261885_ALI_img.png +val/261887_ALI_img.png +val/261891_ALI_img.png +val/261892_ALI_img.png +val/261893_ALI_img.png +val/261905_ALI_img.png +val/261908_ALI_img.png +val/261911_ALI_img.png +val/261915_ALI_img.png +val/261919_ALI_img.png +val/261926_ALI_img.png +val/261933_ALI_img.png +val/261940_ALI_img.png +val/261942_ALI_img.png +val/261945_ALI_img.png +val/261946_ALI_img.png +val/261948_ALI_img.png +val/261949_ALI_img.png +val/261950_ALI_img.png +val/261954_ALI_img.png +val/261955_ALI_img.png +val/261956_ALI_img.png +val/261957_ALI_img.png +val/261961_ALI_img.png +val/261969_ALI_img.png +val/261974_ALI_img.png +val/261977_ALI_img.png +val/261988_ALI_img.png +val/261993_ALI_img.png +val/261995_ALI_img.png +val/261996_ALI_img.png +val/262000_ALI_img.png +val/262001_ALI_img.png +val/262002_ALI_img.png +val/262005_ALI_img.png +val/262007_ALI_img.png +val/262010_ALI_img.png +val/262012_ALI_img.png +val/262017_ALI_img.png +val/262020_ALI_img.png +val/262023_ALI_img.png +val/262025_ALI_img.png +val/262028_ALI_img.png +val/262029_ALI_img.png +val/262034_ALI_img.png +val/262038_ALI_img.png +val/262041_ALI_img.png +val/262042_ALI_img.png +val/262046_ALI_img.png +val/262049_ALI_img.png +val/262051_ALI_img.png +val/262063_ALI_img.png +val/262064_ALI_img.png +val/262065_ALI_img.png +val/262068_ALI_img.png +val/262070_ALI_img.png +val/262070_DT_img.png +val/262076_ALI_img.png +val/262083_ALI_img.png +val/262090_ALI_img.png +val/262091_ALI_img.png +val/262093_ALI_img.png +val/262096_ALI_img.png +val/262097_ALI_img.png +val/262100_ALI_img.png +val/262103_ALI_img.png +val/262104_ALI_img.png +val/262112_ALI_img.png +val/262117_ALI_img.png +val/262122_ALI_img.png +val/262133_ALI_img.png +val/262134_ALI_img.png +val/262136_ALI_img.png +val/262146_ALI_img.png +val/262150_ALI_img.png +val/262153_ALI_img.png +val/262157_ALI_img.png +val/262159_ALI_img.png +val/262163_ALI_img.png +val/262164_ALI_img.png +val/262167_ALI_img.png +val/262170_ALI_img.png +val/262176_ALI_img.png +val/262177_ALI_img.png +val/262178_ALI_img.png +val/262180_ALI_img.png +val/262182_ALI_img.png +val/262195_ALI_img.png +val/262201_ALI_img.png +val/262204_ALI_img.png +val/262208_ALI_img.png +val/262211_ALI_img.png +val/262212_ALI_img.png +val/262215_ALI_img.png +val/262216_ALI_img.png +val/262229_ALI_img.png +val/262231_ALI_img.png +val/262238_ALI_img.png +val/262242_ALI_img.png +val/262244_ALI_img.png +val/262251_ALI_img.png +val/262253_ALI_img.png +val/262261_ALI_img.png +val/262273_ALI_img.png +val/262290_ALI_img.png +val/262294_ALI_img.png +val/262295_ALI_img.png +val/262299_ALI_img.png +val/262302_ALI_img.png +val/262305_ALI_img.png +val/262308_ALI_img.png +val/262309_ALI_img.png +val/262311_ALI_img.png +val/262313_ALI_img.png +val/262323_ALI_img.png +val/262329_ALI_img.png +val/262334_ALI_img.png +val/262339_ALI_img.png +val/262345_ALI_img.png +val/262346_ALI_img.png +val/262349_ALI_img.png +val/262350_ALI_img.png +val/262352_ALI_img.png +val/262358_ALI_img.png +val/262359_ALI_img.png +val/262369_ALI_img.png +val/262380_ALI_img.png +val/262389_ALI_img.png +val/262391_ALI_img.png +val/262394_ALI_img.png +val/262401_ALI_img.png +val/262402_ALI_img.png +val/262410_ALI_img.png +val/262414_ALI_img.png +val/262417_ALI_img.png +val/262420_ALI_img.png +val/262431_ALI_img.png +val/262439_ALI_img.png +val/262440_ALI_img.png +val/262442_ALI_img.png +val/262448_ALI_img.png +val/262450_ALI_img.png +val/262452_ALI_img.png +val/262454_ALI_img.png +val/262456_ALI_img.png +val/262458_ALI_img.png +val/262459_ALI_img.png +val/262460_ALI_img.png +val/262470_ALI_img.png +val/262475_ALI_img.png +val/262478_ALI_img.png +val/262480_ALI_img.png +val/262485_ALI_img.png +val/262494_ALI_img.png +val/262496_ALI_img.png +val/262503_ALI_img.png +val/262507_ALI_img.png +val/262513_ALI_img.png +val/262515_ALI_img.png +val/262520_ALI_img.png +val/262522_ALI_img.png +val/262531_ALI_img.png +val/262535_ALI_img.png +val/262540_ALI_img.png +val/262541_ALI_img.png +val/262542_ALI_img.png +val/262543_ALI_img.png +val/262550_ALI_img.png +val/262553_ALI_img.png +val/262561_ALI_img.png +val/262562_ALI_img.png +val/262565_ALI_img.png +val/262567_ALI_img.png +val/262568_ALI_img.png +val/262574_ALI_img.png +val/262575_ALI_img.png +val/262584_ALI_img.png +val/262587_ALI_img.png +val/262591_ALI_img.png +val/262597_ALI_img.png +val/262598_ALI_img.png +val/262603_ALI_img.png +val/262605_ALI_img.png +val/262609_ALI_img.png +val/262615_ALI_img.png +val/262618_ALI_img.png +val/262626_ALI_img.png +val/262632_ALI_img.png +val/262634_ALI_img.png +val/262638_ALI_img.png +val/262639_ALI_img.png +val/262640_ALI_img.png +val/262642_ALI_img.png +val/262646_ALI_img.png +val/262647_ALI_img.png +val/262648_ALI_img.png +val/262650_ALI_img.png +val/262654_ALI_img.png +val/262655_ALI_img.png +val/262656_ALI_img.png +val/262662_ALI_img.png +val/262673_ALI_img.png +val/262682_ALI_img.png +val/262687_ALI_img.png +val/262692_ALI_img.png +val/262699_ALI_img.png +val/262706_ALI_img.png +val/262711_ALI_img.png +val/262716_ALI_img.png +val/262718_ALI_img.png +val/262720_ALI_img.png +val/262721_ALI_img.png +val/262723_ALI_img.png +val/262727_ALI_img.png +val/262739_ALI_img.png +val/262744_ALI_img.png +val/262755_ALI_img.png +val/262764_ALI_img.png +val/262771_ALI_img.png +val/262777_ALI_img.png +val/262781_ALI_img.png +val/262782_ALI_img.png +val/262783_ALI_img.png +val/262785_ALI_img.png +val/262787_ALI_img.png +val/262789_ALI_img.png +val/262800_ALI_img.png +val/262803_ALI_img.png +val/262804_ALI_img.png +val/262808_ALI_img.png +val/262815_ALI_img.png +val/262816_ALI_img.png +val/262818_ALI_img.png +val/262819_ALI_img.png +val/262835_ALI_img.png +val/262838_ALI_img.png +val/262841_ALI_img.png +val/262842_ALI_img.png +val/262843_ALI_img.png +val/262846_ALI_img.png +val/262848_ALI_img.png +val/262850_ALI_img.png +val/262855_ALI_img.png +val/262857_ALI_img.png +val/262859_ALI_img.png +val/262861_ALI_img.png +val/262869_ALI_img.png +val/262872_ALI_img.png +val/262873_ALI_img.png +val/262874_ALI_img.png +val/262876_ALI_img.png +val/262882_ALI_img.png +val/262884_ALI_img.png +val/262887_ALI_img.png +val/262889_ALI_img.png +val/262891_ALI_img.png +val/262892_ALI_img.png +val/262896_ALI_img.png +val/262899_ALI_img.png +val/262902_ALI_img.png +val/262905_ALI_img.png +val/262910_ALI_img.png +val/262912_ALI_img.png +val/262916_ALI_img.png +val/262924_ALI_img.png +val/262928_ALI_img.png +val/262931_ALI_img.png +val/262933_ALI_img.png +val/262934_ALI_img.png +val/262936_ALI_img.png +val/262937_ALI_img.png +val/262938_ALI_img.png +val/262939_ALI_img.png +val/262941_ALI_img.png +val/262946_ALI_img.png +val/262947_ALI_img.png +val/262948_ALI_img.png +val/262960_ALI_img.png +val/262972_ALI_img.png +val/262984_ALI_img.png +val/262986_ALI_img.png +val/262987_ALI_img.png +val/262989_ALI_img.png +val/263003_ALI_img.png +val/263015_ALI_img.png +val/263019_ALI_img.png +val/263021_ALI_img.png +val/263028_ALI_img.png +val/263029_ALI_img.png +val/263030_ALI_img.png +val/263036_ALI_img.png +val/263048_ALI_img.png +val/263053_ALI_img.png +val/263055_ALI_img.png +val/263057_ALI_img.png +val/263067_ALI_img.png +val/263071_ALI_img.png +val/263073_ALI_img.png +val/263075_ALI_img.png +val/263081_ALI_img.png +val/263087_ALI_img.png +val/263088_ALI_img.png +val/263093_ALI_img.png +val/263099_ALI_img.png +val/263101_ALI_img.png +val/263106_ALI_img.png +val/263108_ALI_img.png +val/263110_ALI_img.png +val/263110_DT_img.png +val/263116_ALI_img.png +val/263118_ALI_img.png +val/263134_ALI_img.png +val/263137_ALI_img.png +val/263154_ALI_img.png +val/263156_ALI_img.png +val/263157_ALI_img.png +val/263159_ALI_img.png +val/263161_ALI_img.png +val/263163_ALI_img.png +val/263167_ALI_img.png +val/263168_ALI_img.png +val/263171_ALI_img.png +val/263172_ALI_img.png +val/263176_ALI_img.png +val/263177_ALI_img.png +val/263184_ALI_img.png +val/263186_ALI_img.png +val/263187_ALI_img.png +val/263190_ALI_img.png +val/263197_ALI_img.png +val/263201_ALI_img.png +val/263204_ALI_img.png +val/263211_ALI_img.png +val/263217_ALI_img.png +val/263221_ALI_img.png +val/263225_ALI_img.png +val/263227_ALI_img.png +val/263228_ALI_img.png +val/263232_ALI_img.png +val/263233_ALI_img.png +val/263236_ALI_img.png +val/263237_ALI_img.png +val/263239_ALI_img.png +val/263257_ALI_img.png +val/263258_ALI_img.png +val/263261_ALI_img.png +val/263269_ALI_img.png +val/263270_DT_img.png +val/263277_ALI_img.png +val/263285_ALI_img.png +val/263292_ALI_img.png +val/263295_ALI_img.png +val/263296_ALI_img.png +val/263298_ALI_img.png +val/263301_ALI_img.png +val/263306_ALI_img.png +val/263310_ALI_img.png +val/263316_ALI_img.png +val/263317_ALI_img.png +val/263319_ALI_img.png +val/263321_ALI_img.png +val/263323_ALI_img.png +val/263324_ALI_img.png +val/263325_ALI_img.png +val/263328_ALI_img.png +val/263330_ALI_img.png +val/263333_ALI_img.png +val/263334_ALI_img.png +val/263344_ALI_img.png +val/263347_ALI_img.png +val/263354_ALI_img.png +val/263355_ALI_img.png +val/263358_ALI_img.png +val/263360_ALI_img.png +val/263367_ALI_img.png +val/263373_ALI_img.png +val/263381_ALI_img.png +val/263384_ALI_img.png +val/263386_ALI_img.png +val/263391_ALI_img.png +val/263401_ALI_img.png +val/263419_ALI_img.png +val/263421_ALI_img.png +val/263423_ALI_img.png +val/263426_ALI_img.png +val/263427_ALI_img.png +val/263432_ALI_img.png +val/263434_ALI_img.png +val/263437_ALI_img.png +val/263440_ALI_img.png +val/263442_ALI_img.png +val/263444_ALI_img.png +val/263446_ALI_img.png +val/263452_ALI_img.png +val/263453_ALI_img.png +val/263454_ALI_img.png +val/263456_ALI_img.png +val/263474_ALI_img.png +val/263475_ALI_img.png +val/263485_ALI_img.png +val/263496_ALI_img.png +val/263513_ALI_img.png +val/263522_ALI_img.png +val/263525_ALI_img.png +val/263528_ALI_img.png +val/263533_ALI_img.png +val/263537_ALI_img.png +val/263539_ALI_img.png +val/263543_ALI_img.png +val/263544_ALI_img.png +val/263548_ALI_img.png +val/263564_ALI_img.png +val/263565_ALI_img.png +val/263574_ALI_img.png +val/263579_ALI_img.png +val/263583_ALI_img.png +val/263599_DT_img.png +val/263602_ALI_img.png +val/263603_ALI_img.png +val/263606_ALI_img.png +val/263607_ALI_img.png +val/263608_ALI_img.png +val/263612_ALI_img.png +val/263613_ALI_img.png +val/263614_ALI_img.png +val/263616_ALI_img.png +val/263620_ALI_img.png +val/263622_ALI_img.png +val/263626_ALI_img.png +val/263631_ALI_img.png +val/263632_ALI_img.png +val/263640_ALI_img.png +val/263642_ALI_img.png +val/263643_ALI_img.png +val/263646_ALI_img.png +val/263647_ALI_img.png +val/263648_ALI_img.png +val/263656_ALI_img.png +val/263658_ALI_img.png +val/263660_ALI_img.png +val/263661_ALI_img.png +val/263662_ALI_img.png +val/263665_ALI_img.png +val/263666_ALI_img.png +val/263669_ALI_img.png +val/263671_ALI_img.png +val/263673_ALI_img.png +val/263676_ALI_img.png +val/263693_ALI_img.png +val/263695_ALI_img.png +val/263704_ALI_img.png +val/263714_ALI_img.png +val/263718_ALI_img.png +val/263721_ALI_img.png +val/263723_ALI_img.png +val/263724_ALI_img.png +val/263727_ALI_img.png +val/263729_ALI_img.png +val/263730_ALI_img.png +val/263731_ALI_img.png +val/263734_ALI_img.png +val/263736_ALI_img.png +val/263737_ALI_img.png +val/263745_ALI_img.png +val/263749_ALI_img.png +val/263751_ALI_img.png +val/263770_ALI_img.png +val/263774_ALI_img.png +val/263775_ALI_img.png +val/263777_ALI_img.png +val/263780_ALI_img.png +val/263782_ALI_img.png +val/263789_ALI_img.png +val/263791_ALI_img.png +val/263794_ALI_img.png +val/263800_ALI_img.png +val/263804_ALI_img.png +val/263811_ALI_img.png +val/263814_ALI_img.png +val/263817_ALI_img.png +val/263820_ALI_img.png +val/263824_ALI_img.png +val/263827_ALI_img.png +val/263834_ALI_img.png +val/263835_ALI_img.png +val/263852_ALI_img.png +val/263854_ALI_img.png +val/263855_ALI_img.png +val/263860_ALI_img.png +val/263861_ALI_img.png +val/263862_ALI_img.png +val/263865_ALI_img.png +val/263868_ALI_img.png +val/263873_ALI_img.png +val/263874_ALI_img.png +val/263876_ALI_img.png +val/263889_ALI_img.png +val/263897_ALI_img.png +val/263898_ALI_img.png +val/263899_ALI_img.png +val/263906_ALI_img.png +val/263913_ALI_img.png +val/263917_ALI_img.png +val/263921_ALI_img.png +val/263926_ALI_img.png +val/263929_ALI_img.png +val/263934_ALI_img.png +val/263943_ALI_img.png +val/263951_ALI_img.png +val/263954_ALI_img.png +val/263958_ALI_img.png +val/263968_ALI_img.png +val/263969_ALI_img.png +val/263970_ALI_img.png +val/263971_ALI_img.png +val/263974_ALI_img.png +val/263976_ALI_img.png +val/263979_ALI_img.png +val/263982_ALI_img.png +val/263983_ALI_img.png +val/263984_ALI_img.png +val/263988_ALI_img.png +val/263990_ALI_img.png +val/263992_ALI_img.png +val/263996_ALI_img.png +val/263998_ALI_img.png +val/264003_ALI_img.png +val/264005_ALI_img.png +val/264011_ALI_img.png +val/264018_ALI_img.png +val/264020_ALI_img.png +val/264022_ALI_img.png +val/264034_ALI_img.png +val/264038_ALI_img.png +val/264043_ALI_img.png +val/264044_ALI_img.png +val/264050_ALI_img.png +val/264053_ALI_img.png +val/264054_ALI_img.png +val/264055_ALI_img.png +val/264057_ALI_img.png +val/264065_ALI_img.png +val/264067_ALI_img.png +val/264068_ALI_img.png +val/264073_ALI_img.png +val/264074_ALI_img.png +val/264082_ALI_img.png +val/264084_ALI_img.png +val/264085_ALI_img.png +val/264087_ALI_img.png +val/264090_ALI_img.png +val/264096_ALI_img.png +val/264099_ALI_img.png +val/264105_ALI_img.png +val/264110_ALI_img.png +val/264114_ALI_img.png +val/264115_ALI_img.png +val/264117_ALI_img.png +val/264118_ALI_img.png +val/264122_ALI_img.png +val/264123_ALI_img.png +val/264125_ALI_img.png +val/264128_ALI_img.png +val/264133_ALI_img.png +val/264136_ALI_img.png +val/264137_ALI_img.png +val/264142_ALI_img.png +val/264144_ALI_img.png +val/264146_ALI_img.png +val/264151_ALI_img.png +val/264155_ALI_img.png +val/264156_ALI_img.png +val/264167_ALI_img.png +val/264169_ALI_img.png +val/264179_ALI_img.png +val/264183_ALI_img.png +val/264184_ALI_img.png +val/264185_ALI_img.png +val/264186_ALI_img.png +val/264193_ALI_img.png +val/264194_ALI_img.png +val/264202_ALI_img.png +val/264204_ALI_img.png +val/264211_ALI_img.png +val/264217_ALI_img.png +val/264220_ALI_img.png +val/264221_ALI_img.png +val/264224_ALI_img.png +val/264227_ALI_img.png +val/264229_ALI_img.png +val/264252_ALI_img.png +val/264253_ALI_img.png +val/264254_ALI_img.png +val/264255_ALI_img.png +val/264256_ALI_img.png +val/264258_ALI_img.png +val/264259_ALI_img.png +val/264262_ALI_img.png +val/264263_ALI_img.png +val/264266_ALI_img.png +val/264269_ALI_img.png +val/264271_ALI_img.png +val/264274_ALI_img.png +val/264277_ALI_img.png +val/264278_ALI_img.png +val/264280_ALI_img.png +val/264285_ALI_img.png +val/264286_ALI_img.png +val/264287_ALI_img.png +val/264294_ALI_img.png +val/264295_ALI_img.png +val/264299_ALI_img.png +val/264301_ALI_img.png +val/264305_ALI_img.png +val/264307_ALI_img.png +val/264311_ALI_img.png +val/264314_ALI_img.png +val/264317_ALI_img.png +val/264318_ALI_img.png +val/264321_ALI_img.png +val/264323_ALI_img.png +val/264327_ALI_img.png +val/264330_ALI_img.png +val/264332_ALI_img.png +val/264335_ALI_img.png +val/264338_ALI_img.png +val/264342_ALI_img.png +val/264343_ALI_img.png +val/264345_ALI_img.png +val/264351_ALI_img.png +val/264352_ALI_img.png +val/264353_ALI_img.png +val/264354_ALI_img.png +val/264364_ALI_img.png +val/264366_ALI_img.png +val/264371_ALI_img.png +val/264372_ALI_img.png +val/264373_ALI_img.png +val/264375_ALI_img.png +val/264376_ALI_img.png +val/264378_ALI_img.png +val/264382_ALI_img.png +val/264383_ALI_img.png +val/264387_ALI_img.png +val/264390_ALI_img.png +val/264391_ALI_img.png +val/264399_ALI_img.png +val/264400_ALI_img.png +val/264408_ALI_img.png +val/264412_ALI_img.png +val/264414_ALI_img.png +val/264415_ALI_img.png +val/264417_ALI_img.png +val/264419_ALI_img.png +val/264424_ALI_img.png +val/264426_ALI_img.png +val/264427_ALI_img.png +val/264430_ALI_img.png +val/264435_ALI_img.png +val/264437_ALI_img.png +val/264438_ALI_img.png +val/264439_ALI_img.png +val/264441_ALI_img.png +val/264443_ALI_img.png +val/264444_ALI_img.png +val/264445_ALI_img.png +val/264446_ALI_img.png +val/264448_ALI_img.png +val/264451_ALI_img.png +val/264453_ALI_img.png +val/264454_ALI_img.png +val/264457_ALI_img.png +val/264458_ALI_img.png +val/264460_ALI_img.png +val/264463_ALI_img.png +val/264464_ALI_img.png +val/264468_ALI_img.png +val/264477_ALI_img.png +val/264480_ALI_img.png +val/264486_ALI_img.png +val/264488_ALI_img.png +val/264489_ALI_img.png +val/264496_ALI_img.png +val/264499_ALI_img.png +val/264501_ALI_img.png +val/264506_ALI_img.png +val/264507_ALI_img.png +val/264509_ALI_img.png +val/264513_ALI_img.png +val/264515_ALI_img.png +val/264516_ALI_img.png +val/264523_ALI_img.png +val/264529_ALI_img.png +val/264532_ALI_img.png +val/264537_ALI_img.png +val/264546_ALI_img.png +val/264552_ALI_img.png +val/264553_ALI_img.png +val/264561_ALI_img.png +val/264563_ALI_img.png +val/264565_ALI_img.png +val/264567_ALI_img.png +val/264572_ALI_img.png +val/264573_ALI_img.png +val/264579_ALI_img.png +val/264586_ALI_img.png +val/264587_ALI_img.png +val/264604_ALI_img.png +val/264606_ALI_img.png +val/264607_ALI_img.png +val/264608_ALI_img.png +val/264609_ALI_img.png +val/264616_ALI_img.png +val/264621_ALI_img.png +val/264622_ALI_img.png +val/264623_ALI_img.png +val/264631_ALI_img.png +val/264633_ALI_img.png +val/264638_ALI_img.png +val/264639_ALI_img.png +val/264646_ALI_img.png +val/264651_ALI_img.png +val/264656_ALI_img.png +val/264659_ALI_img.png +val/264661_ALI_img.png +val/264663_ALI_img.png +val/264664_ALI_img.png +val/264665_ALI_img.png +val/264667_ALI_img.png +val/264668_ALI_img.png +val/264669_ALI_img.png +val/264674_ALI_img.png +val/264682_ALI_img.png +val/264690_ALI_img.png +val/264700_ALI_img.png +val/264709_ALI_img.png +val/264711_ALI_img.png +val/264718_ALI_img.png +val/264719_ALI_img.png +val/264721_ALI_img.png +val/264730_ALI_img.png +val/264734_ALI_img.png +val/264736_ALI_img.png +val/264739_ALI_img.png +val/264746_ALI_img.png +val/264758_ALI_img.png +val/264765_ALI_img.png +val/264769_ALI_img.png +val/264772_ALI_img.png +val/264774_ALI_img.png +val/264779_ALI_img.png +val/264781_ALI_img.png +val/264787_ALI_img.png +val/264788_ALI_img.png +val/264789_ALI_img.png +val/264792_ALI_img.png +val/264797_ALI_img.png +val/264799_ALI_img.png +val/264801_ALI_img.png +val/264803_ALI_img.png +val/264804_ALI_img.png +val/264813_ALI_img.png +val/264814_ALI_img.png +val/264817_ALI_img.png +val/264818_ALI_img.png +val/264825_ALI_img.png +val/264826_ALI_img.png +val/264841_ALI_img.png +val/264845_ALI_img.png +val/264849_ALI_img.png +val/264861_ALI_img.png +val/264862_ALI_img.png +val/264863_ALI_img.png +val/264868_ALI_img.png +val/264880_ALI_img.png +val/264885_ALI_img.png +val/264887_ALI_img.png +val/264892_ALI_img.png +val/264898_ALI_img.png +val/264901_ALI_img.png +val/264913_ALI_img.png +val/264914_ALI_img.png +val/264915_ALI_img.png +val/264925_ALI_img.png +val/264927_ALI_img.png +val/264928_ALI_img.png +val/264930_ALI_img.png +val/264931_ALI_img.png +val/264932_ALI_img.png +val/264933_ALI_img.png +val/264939_ALI_img.png +val/264955_ALI_img.png +val/264957_ALI_img.png +val/264958_ALI_img.png +val/264968_ALI_img.png +val/264969_ALI_img.png +val/264976_ALI_img.png +val/264977_ALI_img.png +val/264985_ALI_img.png +val/264986_ALI_img.png +val/264988_ALI_img.png +val/264989_ALI_img.png +val/264997_ALI_img.png +val/265001_ALI_img.png +val/265002_ALI_img.png +val/265004_ALI_img.png +val/265006_ALI_img.png +val/265011_ALI_img.png +val/265016_ALI_img.png +val/265022_ALI_img.png +val/265023_ALI_img.png +val/265025_ALI_img.png +val/265030_ALI_img.png +val/265033_ALI_img.png +val/265034_ALI_img.png +val/265035_ALI_img.png +val/265039_ALI_img.png +val/265040_ALI_img.png +val/265041_ALI_img.png +val/265045_ALI_img.png +val/265048_ALI_img.png +val/265049_ALI_img.png +val/265050_ALI_img.png +val/265057_ALI_img.png +val/265067_ALI_img.png +val/265070_ALI_img.png +val/265072_ALI_img.png +val/265075_ALI_img.png +val/265076_ALI_img.png +val/265086_ALI_img.png +val/265093_ALI_img.png +val/265094_ALI_img.png +val/265099_ALI_img.png +val/265101_ALI_img.png +val/265106_ALI_img.png +val/265107_ALI_img.png +val/265108_ALI_img.png +val/265122_ALI_img.png +val/265124_ALI_img.png +val/265126_ALI_img.png +val/265127_ALI_img.png +val/265133_ALI_img.png +val/265135_ALI_img.png +val/265136_ALI_img.png +val/265141_ALI_img.png +val/265144_ALI_img.png +val/265145_ALI_img.png +val/265147_ALI_img.png +val/265148_ALI_img.png +val/265160_ALI_img.png +val/265167_ALI_img.png +val/265179_ALI_img.png +val/265180_ALI_img.png +val/265188_ALI_img.png +val/265198_ALI_img.png +val/265202_ALI_img.png +val/265205_ALI_img.png +val/265206_ALI_img.png +val/265207_ALI_img.png +val/265209_ALI_img.png +val/265211_ALI_img.png +val/265214_ALI_img.png +val/265215_ALI_img.png +val/265218_ALI_img.png +val/265219_ALI_img.png +val/265221_ALI_img.png +val/265225_ALI_img.png +val/265231_ALI_img.png +val/265233_ALI_img.png +val/265238_ALI_img.png +val/265243_ALI_img.png +val/265247_ALI_img.png +val/265251_ALI_img.png +val/265254_ALI_img.png +val/265255_ALI_img.png +val/265262_ALI_img.png +val/265264_ALI_img.png +val/265265_ALI_img.png +val/265266_ALI_img.png +val/265267_ALI_img.png +val/265270_ALI_img.png +val/265277_ALI_img.png +val/265283_ALI_img.png +val/265286_ALI_img.png +val/265287_ALI_img.png +val/265289_ALI_img.png +val/265295_ALI_img.png +val/265296_ALI_img.png +val/265298_ALI_img.png +val/265301_ALI_img.png +val/265305_ALI_img.png +val/265308_ALI_img.png +val/265315_ALI_img.png +val/265317_ALI_img.png +val/265319_ALI_img.png +val/265326_ALI_img.png +val/265331_ALI_img.png +val/265333_ALI_img.png +val/265336_ALI_img.png +val/265337_ALI_img.png +val/265343_ALI_img.png +val/265345_ALI_img.png +val/265351_ALI_img.png +val/265353_ALI_img.png +val/265358_ALI_img.png +val/265360_ALI_img.png +val/265369_ALI_img.png +val/265370_ALI_img.png +val/265371_ALI_img.png +val/265373_ALI_img.png +val/265376_ALI_img.png +val/265377_ALI_img.png +val/265379_ALI_img.png +val/265398_ALI_img.png +val/265401_ALI_img.png +val/265402_ALI_img.png +val/265409_ALI_img.png +val/265410_ALI_img.png +val/265412_ALI_img.png +val/265414_ALI_img.png +val/265415_ALI_img.png +val/265420_ALI_img.png +val/265423_ALI_img.png +val/265424_ALI_img.png +val/265425_ALI_img.png +val/265426_ALI_img.png +val/265432_ALI_img.png +val/265438_ALI_img.png +val/265445_ALI_img.png +val/265450_DT_img.png +val/265454_ALI_img.png +val/265480_ALI_img.png +val/265484_ALI_img.png +val/265487_ALI_img.png +val/265488_ALI_img.png +val/265495_ALI_img.png +val/265495_DT_img.png +val/265514_ALI_img.png +val/265515_ALI_img.png +val/265517_ALI_img.png +val/265521_ALI_img.png +val/265528_ALI_img.png +val/265530_ALI_img.png +val/265532_ALI_img.png +val/265540_ALI_img.png +val/265541_ALI_img.png +val/265542_ALI_img.png +val/265544_ALI_img.png +val/265545_ALI_img.png +val/265547_ALI_img.png +val/265549_ALI_img.png +val/265550_ALI_img.png +val/265552_ALI_img.png +val/265553_ALI_img.png +val/265560_ALI_img.png +val/265563_ALI_img.png +val/265569_ALI_img.png +val/265571_ALI_img.png +val/265572_ALI_img.png +val/265575_ALI_img.png +val/265577_ALI_img.png +val/265580_ALI_img.png +val/265584_ALI_img.png +val/265585_ALI_img.png +val/265588_ALI_img.png +val/265600_ALI_img.png +val/265603_ALI_img.png +val/265608_ALI_img.png +val/265611_ALI_img.png +val/265620_ALI_img.png +val/265622_ALI_img.png +val/265625_ALI_img.png +val/265627_ALI_img.png +val/265632_ALI_img.png +val/265634_ALI_img.png +val/265636_ALI_img.png +val/265639_ALI_img.png +val/265642_ALI_img.png +val/265647_ALI_img.png +val/265651_ALI_img.png +val/265655_ALI_img.png +val/265670_ALI_img.png +val/265679_ALI_img.png +val/265683_ALI_img.png +val/265686_ALI_img.png +val/265687_ALI_img.png +val/265688_ALI_img.png +val/265691_ALI_img.png +val/265697_ALI_img.png +val/265705_ALI_img.png +val/265707_ALI_img.png +val/265708_ALI_img.png +val/265709_ALI_img.png +val/265713_ALI_img.png +val/265715_ALI_img.png +val/265716_ALI_img.png +val/265717_ALI_img.png +val/265718_ALI_img.png +val/265724_ALI_img.png +val/265725_ALI_img.png +val/265726_ALI_img.png +val/265727_ALI_img.png +val/265730_ALI_img.png +val/265744_ALI_img.png +val/265748_ALI_img.png +val/265754_ALI_img.png +val/265755_ALI_img.png +val/265757_ALI_img.png +val/265762_ALI_img.png +val/265767_ALI_img.png +val/265774_ALI_img.png +val/265777_ALI_img.png +val/265778_ALI_img.png +val/265779_ALI_img.png +val/265782_ALI_img.png +val/265783_ALI_img.png +val/265789_ALI_img.png +val/265790_ALI_img.png +val/265795_ALI_img.png +val/265800_ALI_img.png +val/265802_ALI_img.png +val/265803_ALI_img.png +val/265805_ALI_img.png +val/265813_ALI_img.png +val/265821_ALI_img.png +val/265822_ALI_img.png +val/265824_ALI_img.png +val/265826_ALI_img.png +val/265832_ALI_img.png +val/265836_ALI_img.png +val/265837_ALI_img.png +val/265838_ALI_img.png +val/265846_ALI_img.png +val/265848_ALI_img.png +val/265854_ALI_img.png +val/265862_ALI_img.png +val/265863_ALI_img.png +val/265869_ALI_img.png +val/265870_ALI_img.png +val/265872_ALI_img.png +val/265878_ALI_img.png +val/265895_ALI_img.png +val/265902_ALI_img.png +val/265903_ALI_img.png +val/265909_ALI_img.png +val/265912_ALI_img.png +val/265917_ALI_img.png +val/265924_ALI_img.png +val/265925_ALI_img.png +val/265928_ALI_img.png +val/265930_ALI_img.png +val/265938_ALI_img.png +val/265950_ALI_img.png +val/265955_ALI_img.png +val/265962_ALI_img.png +val/265963_ALI_img.png +val/265971_ALI_img.png +val/265973_ALI_img.png +val/265981_ALI_img.png +val/265987_ALI_img.png +val/265988_ALI_img.png +val/265994_ALI_img.png +val/266002_ALI_img.png +val/266013_ALI_img.png +val/266015_ALI_img.png +val/266016_ALI_img.png +val/266018_ALI_img.png +val/266020_ALI_img.png +val/266029_DT_img.png +val/266031_ALI_img.png +val/266036_ALI_img.png +val/266039_ALI_img.png +val/266045_ALI_img.png +val/266046_ALI_img.png +val/266050_ALI_img.png +val/266054_ALI_img.png +val/266063_ALI_img.png +val/266068_ALI_img.png +val/266073_ALI_img.png +val/266085_ALI_img.png +val/266091_ALI_img.png +val/266094_ALI_img.png +val/266107_ALI_img.png +val/266111_ALI_img.png +val/266126_ALI_img.png +val/266136_ALI_img.png +val/266139_ALI_img.png +val/266143_ALI_img.png +val/266148_ALI_img.png +val/266153_ALI_img.png +val/266160_ALI_img.png +val/266163_ALI_img.png +val/266165_ALI_img.png +val/266166_ALI_img.png +val/266173_ALI_img.png +val/266183_ALI_img.png +val/266185_ALI_img.png +val/266190_ALI_img.png +val/266192_ALI_img.png +val/266196_ALI_img.png +val/266199_ALI_img.png +val/266208_ALI_img.png +val/266211_ALI_img.png +val/266214_ALI_img.png +val/266215_ALI_img.png +val/266216_ALI_img.png +val/266220_ALI_img.png +val/266222_ALI_img.png +val/266225_ALI_img.png +val/266230_ALI_img.png +val/266231_ALI_img.png +val/266233_ALI_img.png +val/266234_ALI_img.png +val/266238_ALI_img.png +val/266239_ALI_img.png +val/266240_ALI_img.png +val/266243_ALI_img.png +val/266251_ALI_img.png +val/266252_ALI_img.png +val/266257_ALI_img.png +val/266258_ALI_img.png +val/266265_ALI_img.png +val/266267_ALI_img.png +val/266269_ALI_img.png +val/266275_ALI_img.png +val/266277_ALI_img.png +val/266281_ALI_img.png +val/266286_ALI_img.png +val/266288_ALI_img.png +val/266295_ALI_img.png +val/266296_ALI_img.png +val/266297_ALI_img.png +val/266298_ALI_img.png +val/266300_ALI_img.png +val/266304_ALI_img.png +val/266305_ALI_img.png +val/266309_ALI_img.png +val/266311_ALI_img.png +val/266325_ALI_img.png +val/266329_ALI_img.png +val/266337_ALI_img.png +val/266340_ALI_img.png +val/266348_ALI_img.png +val/266349_ALI_img.png +val/266350_ALI_img.png +val/266355_ALI_img.png +val/266376_ALI_img.png +val/266379_ALI_img.png +val/266381_ALI_img.png +val/266388_ALI_img.png +val/266392_ALI_img.png +val/266393_ALI_img.png +val/266414_ALI_img.png +val/266417_ALI_img.png +val/266436_ALI_img.png +val/266440_ALI_img.png +val/266444_ALI_img.png +val/266447_ALI_img.png +val/266448_ALI_img.png +val/266454_ALI_img.png +val/266456_ALI_img.png +val/266459_ALI_img.png +val/266460_ALI_img.png +val/266463_ALI_img.png +val/266467_ALI_img.png +val/266470_ALI_img.png +val/266478_ALI_img.png +val/266486_ALI_img.png +val/266489_ALI_img.png +val/266492_ALI_img.png +val/266497_ALI_img.png +val/266503_ALI_img.png +val/266504_ALI_img.png +val/266505_ALI_img.png +val/266507_ALI_img.png +val/266515_ALI_img.png +val/266527_ALI_img.png +val/266537_ALI_img.png +val/266538_ALI_img.png +val/266547_ALI_img.png +val/266548_ALI_img.png +val/266565_ALI_img.png +val/266570_ALI_img.png +val/266576_ALI_img.png +val/266577_ALI_img.png +val/266580_ALI_img.png +val/266588_ALI_img.png +val/266594_ALI_img.png +val/266596_ALI_img.png +val/266597_ALI_img.png +val/266602_ALI_img.png +val/266607_ALI_img.png +val/266609_ALI_img.png +val/266610_ALI_img.png +val/266611_ALI_img.png +val/266613_ALI_img.png +val/266615_ALI_img.png +val/266620_ALI_img.png +val/266625_ALI_img.png +val/266628_ALI_img.png +val/266629_ALI_img.png +val/266632_ALI_img.png +val/266635_ALI_img.png +val/266636_ALI_img.png +val/266641_ALI_img.png +val/266643_ALI_img.png +val/266648_ALI_img.png +val/266654_ALI_img.png +val/266667_ALI_img.png +val/266668_ALI_img.png +val/266669_ALI_img.png +val/266670_ALI_img.png +val/266671_ALI_img.png +val/266686_ALI_img.png +val/266688_ALI_img.png +val/266691_ALI_img.png +val/266697_ALI_img.png +val/266702_ALI_img.png +val/266704_ALI_img.png +val/266714_ALI_img.png +val/266716_ALI_img.png +val/266729_ALI_img.png +val/266734_ALI_img.png +val/266761_ALI_img.png +val/266766_ALI_img.png +val/266774_ALI_img.png +val/266780_ALI_img.png +val/266782_ALI_img.png +val/266792_ALI_img.png +val/266794_ALI_img.png +val/266800_ALI_img.png +val/266810_ALI_img.png +val/266813_ALI_img.png +val/266821_ALI_img.png +val/266835_ALI_img.png +val/266836_ALI_img.png +val/266837_ALI_img.png +val/266841_ALI_img.png +val/266852_ALI_img.png +val/266856_ALI_img.png +val/266859_ALI_img.png +val/266860_ALI_img.png +val/266862_ALI_img.png +val/266864_ALI_img.png +val/266865_ALI_img.png +val/266877_ALI_img.png +val/266878_ALI_img.png +val/266881_ALI_img.png +val/266884_ALI_img.png +val/266885_ALI_img.png +val/266896_ALI_img.png +val/266904_ALI_img.png +val/266907_ALI_img.png +val/266919_ALI_img.png +val/266920_ALI_img.png +val/266922_ALI_img.png +val/266926_ALI_img.png +val/266927_ALI_img.png +val/266933_ALI_img.png +val/266946_ALI_img.png +val/266949_ALI_img.png +val/266958_ALI_img.png +val/266959_ALI_img.png +val/266963_ALI_img.png +val/266964_ALI_img.png +val/266965_ALI_img.png +val/266970_ALI_img.png +val/266976_ALI_img.png +val/266979_ALI_img.png +val/266980_ALI_img.png +val/266988_ALI_img.png +val/266991_ALI_img.png +val/266995_ALI_img.png +val/266996_ALI_img.png +val/267000_ALI_img.png +val/267004_ALI_img.png +val/267008_ALI_img.png +val/267009_ALI_img.png +val/267030_ALI_img.png +val/267032_ALI_img.png +val/267045_ALI_img.png +val/267093_ALI_img.png +val/267117_ALI_img.png +val/267122_ALI_img.png +val/267133_ALI_img.png +val/267136_ALI_img.png +val/267138_ALI_img.png +val/267140_ALI_img.png +val/267146_ALI_img.png +val/267148_ALI_img.png +val/267157_ALI_img.png +val/267158_ALI_img.png +val/267166_ALI_img.png +val/267175_ALI_img.png +val/267177_ALI_img.png +val/267178_ALI_img.png +val/267180_ALI_img.png +val/267186_ALI_img.png +val/267190_ALI_img.png +val/267191_ALI_img.png +val/267193_ALI_img.png +val/267194_ALI_img.png +val/267195_ALI_img.png +val/267197_ALI_img.png +val/267206_ALI_img.png +val/267208_ALI_img.png +val/267210_ALI_img.png +val/267224_ALI_img.png +val/267226_ALI_img.png +val/267227_ALI_img.png +val/267228_ALI_img.png +val/267237_ALI_img.png +val/267239_ALI_img.png +val/267241_ALI_img.png +val/267246_ALI_img.png +val/267265_ALI_img.png +val/267268_ALI_img.png +val/267277_ALI_img.png +val/267282_ALI_img.png +val/267283_ALI_img.png +val/267286_ALI_img.png +val/267288_ALI_img.png +val/267293_ALI_img.png +val/267311_ALI_img.png +val/267316_ALI_img.png +val/267320_ALI_img.png +val/267323_ALI_img.png +val/267331_ALI_img.png +val/267333_ALI_img.png +val/267335_ALI_img.png +val/267336_ALI_img.png +val/267340_ALI_img.png +val/267345_ALI_img.png +val/267349_ALI_img.png +val/267355_ALI_img.png +val/267362_ALI_img.png +val/267371_ALI_img.png +val/267377_ALI_img.png +val/267382_ALI_img.png +val/267402_ALI_img.png +val/267404_ALI_img.png +val/267407_ALI_img.png +val/267414_ALI_img.png +val/267415_ALI_img.png +val/267421_ALI_img.png +val/267423_ALI_img.png +val/267424_ALI_img.png +val/267426_ALI_img.png +val/267428_ALI_img.png +val/267431_ALI_img.png +val/267433_ALI_img.png +val/267439_ALI_img.png +val/267441_ALI_img.png +val/267443_ALI_img.png +val/267457_ALI_img.png +val/267463_ALI_img.png +val/267469_ALI_img.png +val/267474_ALI_img.png +val/267475_ALI_img.png +val/267480_ALI_img.png +val/267488_ALI_img.png +val/267490_ALI_img.png +val/267492_ALI_img.png +val/267498_ALI_img.png +val/267500_ALI_img.png +val/267501_ALI_img.png +val/267504_ALI_img.png +val/267508_ALI_img.png +val/267509_ALI_img.png +val/267513_ALI_img.png +val/267516_ALI_img.png +val/267518_ALI_img.png +val/267520_ALI_img.png +val/267522_ALI_img.png +val/267527_ALI_img.png +val/267536_ALI_img.png +val/267545_ALI_img.png +val/267547_ALI_img.png +val/267548_ALI_img.png +val/267550_ALI_img.png +val/267551_ALI_img.png +val/267553_ALI_img.png +val/267558_ALI_img.png +val/267566_ALI_img.png +val/267580_ALI_img.png +val/267585_ALI_img.png +val/267587_ALI_img.png +val/267588_ALI_img.png +val/267599_ALI_img.png +val/267601_ALI_img.png +val/267603_ALI_img.png +val/267622_ALI_img.png +val/267626_ALI_img.png +val/267629_ALI_img.png +val/267630_ALI_img.png +val/267633_ALI_img.png +val/267634_ALI_img.png +val/267637_ALI_img.png +val/267645_ALI_img.png +val/267649_ALI_img.png +val/267651_ALI_img.png +val/267668_ALI_img.png +val/267669_ALI_img.png +val/267672_ALI_img.png +val/267686_ALI_img.png +val/267695_ALI_img.png +val/267699_ALI_img.png +val/267700_ALI_img.png +val/267707_ALI_img.png +val/267719_ALI_img.png +val/267723_ALI_img.png +val/267729_ALI_img.png +val/267734_ALI_img.png +val/267736_ALI_img.png +val/267743_ALI_img.png +val/267754_ALI_img.png +val/267762_ALI_img.png +val/267763_ALI_img.png +val/267764_ALI_img.png +val/267769_ALI_img.png +val/267771_ALI_img.png +val/267775_ALI_img.png +val/267781_ALI_img.png +val/267783_ALI_img.png +val/267784_ALI_img.png +val/267790_ALI_img.png +val/267796_ALI_img.png +val/267803_ALI_img.png +val/267807_ALI_img.png +val/267809_ALI_img.png +val/267811_ALI_img.png +val/267814_ALI_img.png +val/267817_ALI_img.png +val/267828_ALI_img.png +val/267831_ALI_img.png +val/267833_ALI_img.png +val/267838_ALI_img.png +val/267860_ALI_img.png +val/267868_ALI_img.png +val/267871_ALI_img.png +val/267874_ALI_img.png +val/267881_ALI_img.png +val/267887_ALI_img.png +val/267888_ALI_img.png +val/267890_ALI_img.png +val/267892_ALI_img.png +val/267894_ALI_img.png +val/267896_ALI_img.png +val/267899_ALI_img.png +val/267910_ALI_img.png +val/267912_ALI_img.png +val/267915_ALI_img.png +val/267920_ALI_img.png +val/267938_ALI_img.png +val/267939_ALI_img.png +val/267947_ALI_img.png +val/267956_ALI_img.png +val/267961_ALI_img.png +val/267962_ALI_img.png +val/267963_ALI_img.png +val/267969_ALI_img.png +val/267981_ALI_img.png +val/267986_ALI_img.png +val/267991_ALI_img.png +val/267998_ALI_img.png +val/268002_ALI_img.png +val/268010_ALI_img.png +val/268014_ALI_img.png +val/268026_ALI_img.png +val/268042_ALI_img.png +val/268045_ALI_img.png +val/268046_ALI_img.png +val/268050_ALI_img.png +val/268056_ALI_img.png +val/268062_ALI_img.png +val/268064_ALI_img.png +val/268067_ALI_img.png +val/268068_ALI_img.png +val/268072_ALI_img.png +val/268075_ALI_img.png +val/268076_ALI_img.png +val/268079_ALI_img.png +val/268081_ALI_img.png +val/268082_ALI_img.png +val/268083_ALI_img.png +val/268084_ALI_img.png +val/268093_ALI_img.png +val/268094_ALI_img.png +val/268101_ALI_img.png +val/268104_ALI_img.png +val/268106_ALI_img.png +val/268107_ALI_img.png +val/268110_ALI_img.png +val/268111_ALI_img.png +val/268122_ALI_img.png +val/268125_ALI_img.png +val/268133_ALI_img.png +val/268141_ALI_img.png +val/268144_ALI_img.png +val/268145_ALI_img.png +val/268147_ALI_img.png +val/268149_ALI_img.png +val/268162_ALI_img.png +val/268165_ALI_img.png +val/268166_ALI_img.png +val/268173_ALI_img.png +val/268183_ALI_img.png +val/268184_ALI_img.png +val/268190_ALI_img.png +val/268194_ALI_img.png +val/268196_ALI_img.png +val/268206_ALI_img.png +val/268208_ALI_img.png +val/268222_ALI_img.png +val/268225_ALI_img.png +val/268238_ALI_img.png +val/268245_ALI_img.png +val/268247_ALI_img.png +val/268251_ALI_img.png +val/268255_ALI_img.png +val/268262_ALI_img.png +val/268263_ALI_img.png +val/268267_ALI_img.png +val/268270_ALI_img.png +val/268271_ALI_img.png +val/268279_ALI_img.png +val/268282_ALI_img.png +val/268285_ALI_img.png +val/268290_ALI_img.png +val/268291_ALI_img.png +val/268295_ALI_img.png +val/268299_ALI_img.png +val/268300_ALI_img.png +val/268302_ALI_img.png +val/268315_ALI_img.png +val/268317_ALI_img.png +val/268325_ALI_img.png +val/268326_ALI_img.png +val/268327_ALI_img.png +val/268330_ALI_img.png +val/268336_ALI_img.png +val/268348_ALI_img.png +val/268352_ALI_img.png +val/268353_ALI_img.png +val/268378_ALI_img.png +val/268382_ALI_img.png +val/268385_ALI_img.png +val/268386_ALI_img.png +val/268387_ALI_img.png +val/268393_ALI_img.png +val/268398_ALI_img.png +val/268399_ALI_img.png +val/268414_ALI_img.png +val/268421_ALI_img.png +val/268423_ALI_img.png +val/268424_ALI_img.png +val/268429_ALI_img.png +val/268430_ALI_img.png +val/268433_ALI_img.png +val/268437_ALI_img.png +val/268440_ALI_img.png +val/268446_ALI_img.png +val/268447_ALI_img.png +val/268450_ALI_img.png +val/268453_ALI_img.png +val/268472_ALI_img.png +val/268473_ALI_img.png +val/268478_ALI_img.png +val/268481_ALI_img.png +val/268483_ALI_img.png +val/268484_ALI_img.png +val/268486_ALI_img.png +val/268489_ALI_img.png +val/268493_ALI_img.png +val/268498_ALI_img.png +val/268500_ALI_img.png +val/268502_ALI_img.png +val/268503_ALI_img.png +val/268504_ALI_img.png +val/268512_ALI_img.png +val/268516_ALI_img.png +val/268519_ALI_img.png +val/268530_ALI_img.png +val/268533_ALI_img.png +val/268534_ALI_img.png +val/268546_ALI_img.png +val/268550_ALI_img.png +val/268552_ALI_img.png +val/268557_ALI_img.png +val/268572_ALI_img.png +val/268584_ALI_img.png +val/268596_ALI_img.png +val/268601_ALI_img.png +val/268620_ALI_img.png +val/268633_ALI_img.png +val/268639_ALI_img.png +val/268642_ALI_img.png +val/268650_ALI_img.png +val/268662_ALI_img.png +val/268665_ALI_img.png +val/268674_ALI_img.png +val/268679_ALI_img.png +val/268680_ALI_img.png +val/268682_ALI_img.png +val/268684_ALI_img.png +val/268694_ALI_img.png +val/268696_ALI_img.png +val/268697_ALI_img.png +val/268699_ALI_img.png +val/268702_ALI_img.png +val/268706_ALI_img.png +val/268714_ALI_img.png +val/268721_ALI_img.png +val/268724_ALI_img.png +val/268735_ALI_img.png +val/268750_ALI_img.png +val/268758_ALI_img.png +val/268766_ALI_img.png +val/268769_ALI_img.png +val/268770_ALI_img.png +val/268789_ALI_img.png +val/268790_ALI_img.png +val/268793_ALI_img.png +val/268799_ALI_img.png +val/268805_ALI_img.png +val/268806_ALI_img.png +val/268810_ALI_img.png +val/268811_ALI_img.png +val/268813_ALI_img.png +val/268821_ALI_img.png +val/268826_ALI_img.png +val/268829_ALI_img.png +val/268845_ALI_img.png +val/268848_ALI_img.png +val/268855_ALI_img.png +val/268858_ALI_img.png +val/268861_ALI_img.png +val/268879_ALI_img.png +val/268880_ALI_img.png +val/268881_ALI_img.png +val/268882_ALI_img.png +val/268887_ALI_img.png +val/268893_ALI_img.png +val/268896_ALI_img.png +val/268901_ALI_img.png +val/268907_ALI_img.png +val/268924_ALI_img.png +val/268926_ALI_img.png +val/268933_ALI_img.png +val/268936_ALI_img.png +val/268943_ALI_img.png +val/268953_ALI_img.png +val/268954_ALI_img.png +val/268955_ALI_img.png +val/268957_ALI_img.png +val/268958_ALI_img.png +val/268959_ALI_img.png +val/268963_ALI_img.png +val/268965_ALI_img.png +val/268968_ALI_img.png +val/268970_ALI_img.png +val/268972_ALI_img.png +val/268974_ALI_img.png +val/268975_ALI_img.png +val/268978_ALI_img.png +val/268981_ALI_img.png +val/268991_ALI_img.png +val/268993_ALI_img.png +val/268996_ALI_img.png +val/269003_ALI_img.png +val/269008_ALI_img.png +val/269010_ALI_img.png +val/269012_ALI_img.png +val/269022_ALI_img.png +val/269029_ALI_img.png +val/269034_ALI_img.png +val/269035_ALI_img.png +val/269049_ALI_img.png +val/269056_ALI_img.png +val/269057_ALI_img.png +val/269069_ALI_img.png +val/269070_ALI_img.png +val/269086_ALI_img.png +val/269092_ALI_img.png +val/269094_ALI_img.png +val/269107_ALI_img.png +val/269108_ALI_img.png +val/269124_ALI_img.png +val/269133_ALI_img.png +val/269136_ALI_img.png +val/269143_ALI_img.png +val/269146_ALI_img.png +val/269150_ALI_img.png +val/269153_ALI_img.png +val/269162_ALI_img.png +val/269166_ALI_img.png +val/269176_ALI_img.png +val/269177_ALI_img.png +val/269181_ALI_img.png +val/269186_ALI_img.png +val/269202_ALI_img.png +val/269209_ALI_img.png +val/269210_ALI_img.png +val/269214_ALI_img.png +val/269229_ALI_img.png +val/269232_ALI_img.png +val/269234_ALI_img.png +val/269235_ALI_img.png +val/269240_ALI_img.png +val/269245_ALI_img.png +val/269250_ALI_img.png +val/269253_ALI_img.png +val/269259_ALI_img.png +val/269260_ALI_img.png +val/269268_ALI_img.png +val/269269_ALI_img.png +val/269272_ALI_img.png +val/269273_ALI_img.png +val/269278_ALI_img.png +val/269286_ALI_img.png +val/269297_ALI_img.png +val/269314_ALI_img.png +val/269333_ALI_img.png +val/269341_ALI_img.png +val/269348_ALI_img.png +val/269350_ALI_img.png +val/269354_ALI_img.png +val/269355_ALI_img.png +val/269378_ALI_img.png +val/269385_ALI_img.png +val/269387_ALI_img.png +val/269392_ALI_img.png +val/269398_ALI_img.png +val/269420_ALI_img.png +val/269424_ALI_img.png +val/269435_ALI_img.png +val/269436_ALI_img.png +val/269437_ALI_img.png +val/269444_ALI_img.png +val/269447_ALI_img.png +val/269449_ALI_img.png +val/269450_ALI_img.png +val/269454_ALI_img.png +val/269459_ALI_img.png +val/269460_ALI_img.png +val/269499_ALI_img.png +val/269502_ALI_img.png +val/269515_ALI_img.png +val/269518_ALI_img.png +val/269527_ALI_img.png +val/269530_ALI_img.png +val/269531_ALI_img.png +val/269533_ALI_img.png +val/269535_ALI_img.png +val/269541_ALI_img.png +val/269546_ALI_img.png +val/269550_ALI_img.png +val/269552_ALI_img.png +val/269553_ALI_img.png +val/269559_ALI_img.png +val/269562_ALI_img.png +val/269563_ALI_img.png +val/269567_ALI_img.png +val/269573_ALI_img.png +val/269575_ALI_img.png +val/269576_ALI_img.png +val/269577_ALI_img.png +val/269581_ALI_img.png +val/269591_ALI_img.png +val/269595_ALI_img.png +val/269599_ALI_img.png +val/269602_ALI_img.png +val/269604_ALI_img.png +val/269607_ALI_img.png +val/269608_ALI_img.png +val/269621_ALI_img.png +val/269625_ALI_img.png +val/269629_ALI_img.png +val/269637_ALI_img.png +val/269638_ALI_img.png +val/269642_ALI_img.png +val/269650_ALI_img.png +val/269651_ALI_img.png +val/269652_ALI_img.png +val/269657_ALI_img.png +val/269659_ALI_img.png +val/269660_ALI_img.png +val/269682_ALI_img.png +val/269689_ALI_img.png +val/269701_ALI_img.png +val/269705_ALI_img.png +val/269707_ALI_img.png +val/269727_ALI_img.png +val/269730_ALI_img.png +val/269744_ALI_img.png +val/269748_ALI_img.png +val/269752_ALI_img.png +val/269754_ALI_img.png +val/269763_ALI_img.png +val/269764_ALI_img.png +val/269766_ALI_img.png +val/269767_ALI_img.png +val/269775_ALI_img.png +val/269777_ALI_img.png +val/269779_ALI_img.png +val/269780_ALI_img.png +val/269793_ALI_img.png +val/269794_ALI_img.png +val/269799_ALI_img.png +val/269800_ALI_img.png +val/269801_ALI_img.png +val/269813_ALI_img.png +val/269816_ALI_img.png +val/269825_ALI_img.png +val/269826_ALI_img.png +val/269831_ALI_img.png +val/269838_ALI_img.png +val/269842_ALI_img.png +val/269851_ALI_img.png +val/269854_ALI_img.png +val/269855_ALI_img.png +val/269857_ALI_img.png +val/269864_ALI_img.png +val/269866_ALI_img.png +val/269869_ALI_img.png +val/269874_ALI_img.png +val/269876_ALI_img.png +val/269879_ALI_img.png +val/269881_ALI_img.png +val/269885_ALI_img.png +val/269891_ALI_img.png +val/269892_ALI_img.png +val/269893_ALI_img.png +val/269894_ALI_img.png +val/269897_ALI_img.png +val/269899_ALI_img.png +val/269903_ALI_img.png +val/269909_ALI_img.png +val/269911_ALI_img.png +val/269914_ALI_img.png +val/269916_ALI_img.png +val/269919_ALI_img.png +val/269930_ALI_img.png +val/269934_ALI_img.png +val/269936_ALI_img.png +val/269940_ALI_img.png +val/269957_ALI_img.png +val/269963_ALI_img.png +val/269966_ALI_img.png +val/269969_ALI_img.png +val/269976_ALI_img.png +val/269980_ALI_img.png +val/269989_ALI_img.png +val/269993_ALI_img.png +val/269994_ALI_img.png +val/269998_ALI_img.png +val/270000_ALI_img.png +val/270001_ALI_img.png +val/270006_ALI_img.png +val/270013_ALI_img.png +val/270015_ALI_img.png +val/270018_ALI_img.png +val/270020_ALI_img.png +val/270021_ALI_img.png +val/270025_ALI_img.png +val/270026_ALI_img.png +val/270043_ALI_img.png +val/270052_ALI_img.png +val/270053_ALI_img.png +val/270054_ALI_img.png +val/270056_ALI_img.png +val/270062_ALI_img.png +val/270066_ALI_img.png +val/270067_ALI_img.png +val/270074_ALI_img.png +val/270078_ALI_img.png +val/270079_ALI_img.png +val/270082_ALI_img.png +val/270085_ALI_img.png +val/270087_ALI_img.png +val/270088_ALI_img.png +val/270092_ALI_img.png +val/270099_ALI_img.png +val/270100_ALI_img.png +val/270101_ALI_img.png +val/270103_ALI_img.png +val/270104_ALI_img.png +val/270110_ALI_img.png +val/270112_ALI_img.png +val/270113_ALI_img.png +val/270116_ALI_img.png +val/270124_ALI_img.png +val/270132_ALI_img.png +val/270135_ALI_img.png +val/270144_ALI_img.png +val/270154_ALI_img.png +val/270155_ALI_img.png +val/270161_ALI_img.png +val/270162_ALI_img.png +val/270176_ALI_img.png +val/270178_ALI_img.png +val/270199_ALI_img.png +val/270204_ALI_img.png +val/270233_ALI_img.png +val/270730_ALI_img.png +val/270972_ALI_img.png +val/271077_ALI_img.png +val/271275_ALI_img.png +val/271333_ALI_img.png +val/271867_ALI_img.png +val/272602_ALI_img.png +val/273419_ALI_img.png +val/274002_ALI_img.png +val/274860_ALI_img.png +val/275901_ALI_img.png +val/276571_ALI_img.png +val/277831_ALI_img.png +val/277967_ALI_img.png +val/278835_ALI_img.png +val/279242_ALI_img.png +val/279283_ALI_img.png +val/279347_ALI_img.png +val/279544_ALI_img.png +val/282086_ALI_img.png +val/282105_ALI_img.png +val/282185_ALI_img.png +val/282390_ALI_img.png +val/283177_ALI_img.png +val/284880_ALI_img.png +val/285331_ALI_img.png +val/285480_ALI_img.png +val/285677_ALI_img.png +val/285863_ALI_img.png +val/286181_ALI_img.png +val/286199_ALI_img.png +val/286377_ALI_img.png +val/288267_ALI_img.png +val/288607_ALI_img.png +val/289101_ALI_img.png +val/289145_ALI_img.png +val/289363_ALI_img.png +val/289419_ALI_img.png +val/289874_ALI_img.png +val/289938_ALI_img.png +val/290891_ALI_img.png +val/291366_ALI_img.png +val/291388_ALI_img.png +val/291611_ALI_img.png +val/291703_ALI_img.png +val/292739_ALI_img.png +val/292887_ALI_img.png +val/293165_ALI_img.png +val/293392_ALI_img.png +val/293789_ALI_img.png +val/294561_ALI_img.png +val/295985_ALI_img.png +val/296350_ALI_img.png +val/296721_ALI_img.png +val/297546_ALI_img.png +val/297612_ALI_img.png +val/297754_ALI_img.png +val/297771_ALI_img.png +val/297917_ALI_img.png +val/298353_ALI_img.png +val/73717_DT_img.png +val/73771_DT_img.png +val/75799_DT_img.png +val/78120_DT_img.png +val/79685_DT_img.png +val/80847_DT_img.png +val/83994_DT_img.png +val/84262_DT_img.png +val/84335_DT_img.png +val/87932_DT_img.png +val/90896_DT_img.png +val/92174_DT_img.png +val/93302_DT_img.png +val/93871_DT_img.png +val/94887_DT_img.png +val/95307_DT_img.png +val/97627_DT_img.png +val/98901_DT_img.png +val/99353_DT_img.png +val/99660_DT_img.png \ No newline at end of file diff --git a/infer/dataset_normal/scannet/__init__.py b/infer/dataset_normal/scannet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8dc97641a9e632e4c5891a13b8fda645e974b347 --- /dev/null +++ b/infer/dataset_normal/scannet/__init__.py @@ -0,0 +1,64 @@ +""" Get samples from ScanNet (https://github.com/ScanNet/ScanNet) + NOTE: GT surface normals and data split are from FrameNet (ICCV 2019) - https://github.com/hjwdzh/FrameNet +""" +import os +import cv2 +import numpy as np + +from infer.dataset_normal import Sample + + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "scene0532_00/000000_img.png" + scene_name = sample_path.split('/')[0] + img_name, img_ext = sample_path.split('/')[1].split('_img') + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'scannet') + img_path = '%s/%s' % (dataset_path, sample_path) + normal_png_path = img_path.replace('_img'+img_ext, '_normal.png') + normal_npy_path = img_path.replace('_img'+img_ext, '_normal.npy') + intrins_path = img_path.replace('_img'+img_ext, '_intrins.npy') + assert os.path.exists(img_path) + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + + # read normal (H, W, 3) + if os.path.exists(normal_png_path): + normal = cv2.cvtColor(cv2.imread(normal_png_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + normal_mask = np.sum(normal, axis=2, keepdims=True) > 0 + normal = (normal.astype(np.float32) / 255.0) * 2.0 - 1.0 + elif os.path.exists(normal_npy_path): + normal = np.load(normal_npy_path).astype(np.float32) + assert normal.ndim == 3 and normal.shape[2] == 3, f"Unexpected normal shape: {normal.shape}" + # FrameNet npy normals use opposite x-axis convention for this evaluation codepath. + normal[:, :, 0] *= -1.0 + normal_mask = np.linalg.norm(normal, axis=2, keepdims=True) > 1e-6 + else: + raise FileNotFoundError(f"Missing ScanNet normal file: {normal_png_path} or {normal_npy_path}") + + # read intrins (3, 3) + if os.path.exists(intrins_path): + intrins = np.load(intrins_path) + else: + # Fallback intrinsics for ScanNet benchmark-sized frames. + intrins = np.array([ + [577.870605, 0.0, 319.5], + [0.0, 577.870605, 239.5], + [0.0, 0.0, 1.0], + ], dtype=np.float32) + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='scannet', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample diff --git a/infer/dataset_normal/scannet/split/test.txt b/infer/dataset_normal/scannet/split/test.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c71d9f0e52ddef397d1158075b20b613caca50c --- /dev/null +++ b/infer/dataset_normal/scannet/split/test.txt @@ -0,0 +1,300 @@ +scene0532_00/000000_img.png +scene0220_02/000450_img.png +scene0061_00/000420_img.png +scene0656_03/000490_img.png +scene0525_01/000960_img.png +scene0310_01/000630_img.png +scene0628_00/000100_img.png +scene0481_01/000900_img.png +scene0608_02/001690_img.png +scene0415_00/000750_img.png +scene0603_00/001590_img.png +scene0740_00/000510_img.png +scene0740_00/002510_img.png +scene0177_00/000550_img.png +scene0395_01/000160_img.png +scene0395_01/002160_img.png +scene0343_00/000210_img.png +scene0241_00/000970_img.png +scene0717_00/000010_img.png +scene0362_00/000890_img.png +scene0457_02/000070_img.png +scene0296_00/000040_img.png +scene0157_00/000300_img.png +scene0502_02/001130_img.png +scene0101_01/000410_img.png +scene0610_02/000730_img.png +scene0610_02/002780_img.png +scene0416_00/000950_img.png +scene0685_02/000840_img.png +scene0685_02/003050_img.png +scene0662_01/001810_img.png +scene0670_01/000840_img.png +scene0670_01/002840_img.png +scene0012_01/000160_img.png +scene0012_01/002160_img.png +scene0320_03/000790_img.png +scene0780_00/000450_img.png +scene0575_01/000510_img.png +scene0575_01/002510_img.png +scene0168_01/000100_img.png +scene0335_02/000980_img.png +scene0691_00/000400_img.png +scene0778_00/001230_img.png +scene0019_01/000490_img.png +scene0554_00/001960_img.png +scene0114_00/000260_img.png +scene0457_01/000350_img.png +scene0548_01/000150_img.png +scene0548_01/002150_img.png +scene0378_00/000610_img.png +scene0505_00/000710_img.png +scene0505_00/002710_img.png +scene0695_01/001200_img.png +scene0424_02/001060_img.png +scene0627_00/001470_img.png +scene0279_00/001200_img.png +scene0291_01/001280_img.png +scene0768_00/000330_img.png +scene0768_00/002330_img.png +scene0696_01/000340_img.png +scene0145_00/000420_img.png +scene0185_00/000370_img.png +scene0185_00/002370_img.png +scene0806_00/001070_img.png +scene0261_03/000410_img.png +scene0068_00/000570_img.png +scene0804_00/000510_img.png +scene0221_01/000490_img.png +scene0630_03/000880_img.png +scene0617_00/001270_img.png +scene0206_02/001640_img.png +scene0582_02/000590_img.png +scene0031_01/000100_img.png +scene0031_01/002100_img.png +scene0505_04/001100_img.png +scene0092_01/000600_img.png +scene0542_00/000570_img.png +scene0525_02/000480_img.png +scene0756_00/000870_img.png +scene0756_00/002870_img.png +scene0061_01/001360_img.png +scene0588_01/000940_img.png +scene0256_02/000610_img.png +scene0300_00/001430_img.png +scene0451_05/001110_img.png +scene0397_01/001390_img.png +scene0372_00/000250_img.png +scene0372_00/002280_img.png +scene0647_01/000790_img.png +scene0474_01/001600_img.png +scene0404_00/000360_img.png +scene0404_00/002360_img.png +scene0702_01/000010_img.png +scene0114_02/001420_img.png +scene0460_00/000820_img.png +scene0151_01/000390_img.png +scene0151_01/002390_img.png +scene0078_02/000520_img.png +scene0772_00/001530_img.png +scene0705_01/000490_img.png +scene0408_01/000590_img.png +scene0538_00/001430_img.png +scene0651_00/000080_img.png +scene0210_01/001040_img.png +scene0002_00/000240_img.png +scene0002_00/002240_img.png +scene0002_00/004240_img.png +scene0347_00/001040_img.png +scene0072_02/000210_img.png +scene0371_00/000410_img.png +scene0211_00/001120_img.png +scene0514_01/000050_img.png +scene0473_01/000910_img.png +scene0362_02/001970_img.png +scene0024_02/001180_img.png +scene0673_00/000280_img.png +scene0176_00/000030_img.png +scene0073_00/000400_img.png +scene0186_01/000230_img.png +scene0236_00/000490_img.png +scene0210_00/000780_img.png +scene0325_01/000920_img.png +scene0044_01/000170_img.png +scene0197_00/001300_img.png +scene0472_02/001190_img.png +scene0611_01/000010_img.png +scene0758_00/000710_img.png +scene0650_00/000870_img.png +scene0728_00/000160_img.png +scene0217_00/000450_img.png +scene0335_01/000070_img.png +scene0335_01/002070_img.png +scene0504_00/001230_img.png +scene0669_01/000780_img.png +scene0143_01/000760_img.png +scene0320_01/000460_img.png +scene0645_01/000340_img.png +scene0645_01/002340_img.png +scene0645_01/004340_img.png +scene0248_01/000300_img.png +scene0416_04/000580_img.png +scene0416_04/002580_img.png +scene0534_00/000780_img.png +scene0489_02/001010_img.png +scene0379_00/000500_img.png +scene0500_01/000480_img.png +scene0561_01/000850_img.png +scene0580_00/001200_img.png +scene0580_00/003200_img.png +scene0552_01/000430_img.png +scene0744_00/000020_img.png +scene0744_00/002020_img.png +scene0044_00/001030_img.png +scene0301_02/000140_img.png +scene0422_00/000940_img.png +scene0655_01/000700_img.png +scene0803_00/001770_img.png +scene0629_02/000530_img.png +scene0609_01/000030_img.png +scene0623_01/000930_img.png +scene0596_00/000660_img.png +scene0150_01/000150_img.png +scene0501_02/001770_img.png +scene0560_00/000420_img.png +scene0406_02/000220_img.png +scene0587_02/001010_img.png +scene0540_00/000730_img.png +scene0074_01/001390_img.png +scene0620_01/000200_img.png +scene0486_00/000320_img.png +scene0486_00/002320_img.png +scene0600_02/001350_img.png +scene0085_00/001800_img.png +scene0084_00/000560_img.png +scene0319_00/000580_img.png +scene0400_01/000890_img.png +scene0548_02/000020_img.png +scene0548_02/002110_img.png +scene0244_01/000360_img.png +scene0785_00/000630_img.png +scene0785_00/002630_img.png +scene0479_02/000530_img.png +scene0121_00/000310_img.png +scene0107_00/000410_img.png +scene0328_00/000040_img.png +scene0196_00/000350_img.png +scene0404_01/000720_img.png +scene0404_01/002720_img.png +scene0081_02/000710_img.png +scene0666_00/000360_img.png +scene0367_00/000330_img.png +scene0340_01/000460_img.png +scene0300_01/000980_img.png +scene0275_00/001230_img.png +scene0036_00/000440_img.png +scene0520_01/001350_img.png +scene0113_01/000000_img.png +scene0541_01/000020_img.png +scene0034_01/001070_img.png +scene0030_01/000160_img.png +scene0438_00/000510_img.png +scene0679_00/001210_img.png +scene0546_00/000400_img.png +scene0223_00/000600_img.png +scene0403_00/001570_img.png +scene0001_01/001170_img.png +scene0014_00/001880_img.png +scene0673_02/001480_img.png +scene0794_00/000000_img.png +scene0209_02/000210_img.png +scene0801_00/000180_img.png +scene0086_00/000800_img.png +scene0501_00/001370_img.png +scene0412_01/000070_img.png +scene0339_00/000730_img.png +scene0724_00/000500_img.png +scene0654_01/001080_img.png +scene0081_01/000460_img.png +scene0576_02/001030_img.png +scene0589_01/001010_img.png +scene0428_00/001470_img.png +scene0199_00/000110_img.png +scene0513_00/000440_img.png +scene0512_00/000110_img.png +scene0508_00/000840_img.png +scene0009_02/000360_img.png +scene0303_00/001250_img.png +scene0533_01/001240_img.png +scene0445_01/000640_img.png +scene0392_00/001870_img.png +scene0111_01/001560_img.png +scene0192_02/000620_img.png +scene0396_00/000170_img.png +scene0376_02/000320_img.png +scene0100_01/000880_img.png +scene0578_01/000920_img.png +scene0765_00/000400_img.png +scene0784_00/000420_img.png +scene0784_00/002430_img.png +scene0784_00/004430_img.png +scene0065_02/000740_img.png +scene0207_02/000040_img.png +scene0207_02/002040_img.png +scene0440_01/000500_img.png +scene0220_01/000450_img.png +scene0713_00/000000_img.png +scene0713_00/002000_img.png +scene0697_02/000070_img.png +scene0697_02/002070_img.png +scene0286_03/001170_img.png +scene0393_02/000810_img.png +scene0370_02/000080_img.png +scene0370_02/002080_img.png +scene0452_02/000950_img.png +scene0226_00/001270_img.png +scene0474_03/001610_img.png +scene0304_00/000600_img.png +scene0250_01/000820_img.png +scene0250_01/002820_img.png +scene0419_02/001480_img.png +scene0548_00/000000_img.png +scene0548_00/002050_img.png +scene0568_00/000150_img.png +scene0324_00/000080_img.png +scene0567_00/000070_img.png +scene0567_00/002070_img.png +scene0659_01/001460_img.png +scene0540_02/000120_img.png +scene0488_01/000680_img.png +scene0146_01/000260_img.png +scene0580_01/001130_img.png +scene0580_01/003130_img.png +scene0421_02/001320_img.png +scene0657_00/000090_img.png +scene0588_02/001380_img.png +scene0471_02/000570_img.png +scene0186_00/000260_img.png +scene0186_00/002260_img.png +scene0065_01/000800_img.png +scene0456_01/000710_img.png +scene0524_01/001680_img.png +scene0062_00/000690_img.png +scene0134_01/000780_img.png +scene0294_00/000280_img.png +scene0294_00/002280_img.png +scene0452_01/001090_img.png +scene0142_00/000770_img.png +scene0135_00/000330_img.png +scene0279_02/000780_img.png +scene0698_01/000390_img.png +scene0520_00/000820_img.png +scene0448_01/000850_img.png +scene0276_00/000320_img.png +scene0731_00/000220_img.png +scene0599_01/001090_img.png +scene0631_02/000210_img.png +scene0748_00/000890_img.png +scene0170_02/000190_img.png +scene0215_01/000400_img.png +scene0178_00/001330_img.png \ No newline at end of file diff --git a/infer/dataset_normal/sintel/__init__.py b/infer/dataset_normal/sintel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..29af73a1b374d3a263890861f0813aae7dfcd21e --- /dev/null +++ b/infer/dataset_normal/sintel/__init__.py @@ -0,0 +1,49 @@ +""" Get samples from Sintel (http://sintel.is.tue.mpg.de/) + NOTE: We computed the GT surface normals by doing discontinuity-aware plane fitting +""" +import os +import cv2 +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" + +from infer.dataset_normal import Sample + + + +def get_sample(base_data_dir, sample_path, info): + # e.g. sample_path = "alley_1/frame_0001_img.png" + scene_name = sample_path.split('/')[0] + img_name, img_ext = sample_path.split('/')[1].split('_img') + + dataset_path = os.path.join(base_data_dir, 'dsine_eval', 'sintel') + img_path = '%s/%s' % (dataset_path, sample_path) + normal_path = img_path.replace('_img'+img_ext, '_normal.exr') + intrins_path = img_path.replace('_img'+img_ext, '_intrins.npy') + assert os.path.exists(img_path) + assert os.path.exists(normal_path) + assert os.path.exists(intrins_path) + + # read image (H, W, 3) + img = cv2.cvtColor(cv2.imread(img_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + img = img.astype(np.float32) / 255.0 + + # read normal (H, W, 3) + normal = cv2.cvtColor(cv2.imread(normal_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB) + normal_mask = np.linalg.norm(normal, axis=2, keepdims=True) > 0.5 + + # read intrins (3, 3) + intrins = np.load(intrins_path) + + sample = Sample( + img=img, + normal=normal, + normal_mask=normal_mask, + intrins=intrins, + + dataset_name='sintel', + scene_name=scene_name, + img_name=img_name, + info=info + ) + + return sample \ No newline at end of file diff --git a/infer/dataset_normal/sintel/split/sintel.txt b/infer/dataset_normal/sintel/split/sintel.txt new file mode 100644 index 0000000000000000000000000000000000000000..03dd29dca88d2be4b36b3177ee5608a627c22840 --- /dev/null +++ b/infer/dataset_normal/sintel/split/sintel.txt @@ -0,0 +1,1064 @@ +alley_1/frame_0001_img.png +alley_1/frame_0002_img.png +alley_1/frame_0003_img.png +alley_1/frame_0004_img.png +alley_1/frame_0005_img.png +alley_1/frame_0006_img.png +alley_1/frame_0007_img.png +alley_1/frame_0008_img.png +alley_1/frame_0009_img.png +alley_1/frame_0010_img.png +alley_1/frame_0011_img.png +alley_1/frame_0012_img.png +alley_1/frame_0013_img.png +alley_1/frame_0014_img.png +alley_1/frame_0015_img.png +alley_1/frame_0016_img.png +alley_1/frame_0017_img.png +alley_1/frame_0018_img.png +alley_1/frame_0019_img.png +alley_1/frame_0020_img.png +alley_1/frame_0021_img.png +alley_1/frame_0022_img.png +alley_1/frame_0023_img.png +alley_1/frame_0024_img.png +alley_1/frame_0025_img.png +alley_1/frame_0026_img.png +alley_1/frame_0027_img.png +alley_1/frame_0028_img.png +alley_1/frame_0029_img.png +alley_1/frame_0030_img.png +alley_1/frame_0031_img.png +alley_1/frame_0032_img.png +alley_1/frame_0033_img.png +alley_1/frame_0034_img.png +alley_1/frame_0035_img.png +alley_1/frame_0036_img.png +alley_1/frame_0037_img.png +alley_1/frame_0038_img.png +alley_1/frame_0039_img.png +alley_1/frame_0040_img.png +alley_1/frame_0041_img.png +alley_1/frame_0042_img.png +alley_1/frame_0043_img.png +alley_1/frame_0044_img.png +alley_1/frame_0045_img.png +alley_1/frame_0046_img.png +alley_1/frame_0047_img.png +alley_1/frame_0048_img.png +alley_1/frame_0049_img.png +alley_1/frame_0050_img.png +alley_2/frame_0001_img.png +alley_2/frame_0002_img.png +alley_2/frame_0003_img.png +alley_2/frame_0004_img.png +alley_2/frame_0005_img.png +alley_2/frame_0006_img.png +alley_2/frame_0007_img.png +alley_2/frame_0008_img.png +alley_2/frame_0009_img.png +alley_2/frame_0010_img.png +alley_2/frame_0011_img.png +alley_2/frame_0012_img.png +alley_2/frame_0013_img.png +alley_2/frame_0014_img.png +alley_2/frame_0015_img.png +alley_2/frame_0016_img.png +alley_2/frame_0017_img.png +alley_2/frame_0018_img.png +alley_2/frame_0019_img.png +alley_2/frame_0020_img.png +alley_2/frame_0021_img.png +alley_2/frame_0022_img.png +alley_2/frame_0023_img.png +alley_2/frame_0024_img.png +alley_2/frame_0025_img.png +alley_2/frame_0026_img.png +alley_2/frame_0027_img.png +alley_2/frame_0028_img.png +alley_2/frame_0029_img.png +alley_2/frame_0030_img.png +alley_2/frame_0031_img.png +alley_2/frame_0032_img.png +alley_2/frame_0033_img.png +alley_2/frame_0034_img.png +alley_2/frame_0035_img.png +alley_2/frame_0036_img.png +alley_2/frame_0037_img.png +alley_2/frame_0038_img.png +alley_2/frame_0039_img.png +alley_2/frame_0040_img.png +alley_2/frame_0041_img.png +alley_2/frame_0042_img.png +alley_2/frame_0043_img.png +alley_2/frame_0044_img.png +alley_2/frame_0045_img.png +alley_2/frame_0046_img.png +alley_2/frame_0047_img.png +alley_2/frame_0048_img.png +alley_2/frame_0049_img.png +alley_2/frame_0050_img.png +ambush_2/frame_0001_img.png +ambush_2/frame_0002_img.png +ambush_2/frame_0003_img.png +ambush_2/frame_0004_img.png +ambush_2/frame_0005_img.png +ambush_2/frame_0006_img.png +ambush_2/frame_0007_img.png +ambush_2/frame_0008_img.png +ambush_2/frame_0009_img.png +ambush_2/frame_0010_img.png +ambush_2/frame_0011_img.png +ambush_2/frame_0012_img.png +ambush_2/frame_0013_img.png +ambush_2/frame_0014_img.png +ambush_2/frame_0015_img.png +ambush_2/frame_0016_img.png +ambush_2/frame_0017_img.png +ambush_2/frame_0018_img.png +ambush_2/frame_0019_img.png +ambush_2/frame_0020_img.png +ambush_2/frame_0021_img.png +ambush_4/frame_0001_img.png +ambush_4/frame_0002_img.png +ambush_4/frame_0003_img.png +ambush_4/frame_0004_img.png +ambush_4/frame_0005_img.png +ambush_4/frame_0006_img.png +ambush_4/frame_0007_img.png +ambush_4/frame_0008_img.png +ambush_4/frame_0009_img.png +ambush_4/frame_0010_img.png +ambush_4/frame_0011_img.png +ambush_4/frame_0012_img.png +ambush_4/frame_0013_img.png +ambush_4/frame_0014_img.png +ambush_4/frame_0015_img.png +ambush_4/frame_0016_img.png +ambush_4/frame_0017_img.png +ambush_4/frame_0018_img.png +ambush_4/frame_0019_img.png +ambush_4/frame_0020_img.png +ambush_4/frame_0021_img.png +ambush_4/frame_0022_img.png +ambush_4/frame_0023_img.png +ambush_4/frame_0024_img.png +ambush_4/frame_0025_img.png +ambush_4/frame_0026_img.png +ambush_4/frame_0027_img.png +ambush_4/frame_0028_img.png +ambush_4/frame_0029_img.png +ambush_4/frame_0030_img.png +ambush_4/frame_0031_img.png +ambush_4/frame_0032_img.png +ambush_4/frame_0033_img.png +ambush_5/frame_0001_img.png +ambush_5/frame_0002_img.png +ambush_5/frame_0003_img.png +ambush_5/frame_0004_img.png +ambush_5/frame_0005_img.png +ambush_5/frame_0006_img.png +ambush_5/frame_0007_img.png +ambush_5/frame_0008_img.png +ambush_5/frame_0009_img.png +ambush_5/frame_0010_img.png +ambush_5/frame_0011_img.png +ambush_5/frame_0012_img.png +ambush_5/frame_0013_img.png +ambush_5/frame_0014_img.png +ambush_5/frame_0015_img.png +ambush_5/frame_0016_img.png +ambush_5/frame_0017_img.png +ambush_5/frame_0018_img.png +ambush_5/frame_0019_img.png +ambush_5/frame_0020_img.png +ambush_5/frame_0021_img.png +ambush_5/frame_0022_img.png +ambush_5/frame_0023_img.png +ambush_5/frame_0024_img.png +ambush_5/frame_0025_img.png +ambush_5/frame_0026_img.png +ambush_5/frame_0027_img.png +ambush_5/frame_0028_img.png +ambush_5/frame_0029_img.png +ambush_5/frame_0030_img.png +ambush_5/frame_0031_img.png +ambush_5/frame_0032_img.png +ambush_5/frame_0033_img.png +ambush_5/frame_0034_img.png +ambush_5/frame_0035_img.png +ambush_5/frame_0036_img.png +ambush_5/frame_0037_img.png +ambush_5/frame_0038_img.png +ambush_5/frame_0039_img.png +ambush_5/frame_0040_img.png +ambush_5/frame_0041_img.png +ambush_5/frame_0042_img.png +ambush_5/frame_0043_img.png +ambush_5/frame_0044_img.png +ambush_5/frame_0045_img.png +ambush_5/frame_0046_img.png +ambush_5/frame_0047_img.png +ambush_5/frame_0048_img.png +ambush_5/frame_0049_img.png +ambush_5/frame_0050_img.png +ambush_6/frame_0001_img.png +ambush_6/frame_0002_img.png +ambush_6/frame_0003_img.png +ambush_6/frame_0004_img.png +ambush_6/frame_0005_img.png +ambush_6/frame_0006_img.png +ambush_6/frame_0007_img.png +ambush_6/frame_0008_img.png +ambush_6/frame_0009_img.png +ambush_6/frame_0010_img.png +ambush_6/frame_0011_img.png +ambush_6/frame_0012_img.png +ambush_6/frame_0013_img.png +ambush_6/frame_0014_img.png +ambush_6/frame_0015_img.png +ambush_6/frame_0016_img.png +ambush_6/frame_0017_img.png +ambush_6/frame_0018_img.png +ambush_6/frame_0019_img.png +ambush_6/frame_0020_img.png +ambush_7/frame_0001_img.png +ambush_7/frame_0002_img.png +ambush_7/frame_0003_img.png +ambush_7/frame_0004_img.png +ambush_7/frame_0005_img.png +ambush_7/frame_0006_img.png +ambush_7/frame_0007_img.png +ambush_7/frame_0008_img.png +ambush_7/frame_0009_img.png +ambush_7/frame_0010_img.png +ambush_7/frame_0011_img.png +ambush_7/frame_0012_img.png +ambush_7/frame_0013_img.png +ambush_7/frame_0014_img.png +ambush_7/frame_0015_img.png +ambush_7/frame_0016_img.png +ambush_7/frame_0017_img.png +ambush_7/frame_0018_img.png +ambush_7/frame_0019_img.png +ambush_7/frame_0020_img.png +ambush_7/frame_0021_img.png +ambush_7/frame_0022_img.png +ambush_7/frame_0023_img.png +ambush_7/frame_0024_img.png +ambush_7/frame_0025_img.png +ambush_7/frame_0026_img.png +ambush_7/frame_0027_img.png +ambush_7/frame_0028_img.png +ambush_7/frame_0029_img.png +ambush_7/frame_0030_img.png +ambush_7/frame_0031_img.png +ambush_7/frame_0032_img.png +ambush_7/frame_0033_img.png +ambush_7/frame_0034_img.png +ambush_7/frame_0035_img.png +ambush_7/frame_0036_img.png +ambush_7/frame_0037_img.png +ambush_7/frame_0038_img.png +ambush_7/frame_0039_img.png +ambush_7/frame_0040_img.png +ambush_7/frame_0041_img.png +ambush_7/frame_0042_img.png +ambush_7/frame_0043_img.png +ambush_7/frame_0044_img.png +ambush_7/frame_0045_img.png +ambush_7/frame_0046_img.png +ambush_7/frame_0047_img.png +ambush_7/frame_0048_img.png +ambush_7/frame_0049_img.png +ambush_7/frame_0050_img.png +bamboo_1/frame_0001_img.png +bamboo_1/frame_0002_img.png +bamboo_1/frame_0003_img.png +bamboo_1/frame_0004_img.png +bamboo_1/frame_0005_img.png +bamboo_1/frame_0006_img.png +bamboo_1/frame_0007_img.png +bamboo_1/frame_0008_img.png +bamboo_1/frame_0009_img.png +bamboo_1/frame_0010_img.png +bamboo_1/frame_0011_img.png +bamboo_1/frame_0012_img.png +bamboo_1/frame_0013_img.png +bamboo_1/frame_0014_img.png +bamboo_1/frame_0015_img.png +bamboo_1/frame_0016_img.png +bamboo_1/frame_0017_img.png +bamboo_1/frame_0018_img.png +bamboo_1/frame_0019_img.png +bamboo_1/frame_0020_img.png +bamboo_1/frame_0021_img.png +bamboo_1/frame_0022_img.png +bamboo_1/frame_0023_img.png +bamboo_1/frame_0024_img.png +bamboo_1/frame_0025_img.png +bamboo_1/frame_0026_img.png +bamboo_1/frame_0027_img.png +bamboo_1/frame_0028_img.png +bamboo_1/frame_0029_img.png +bamboo_1/frame_0030_img.png +bamboo_1/frame_0031_img.png +bamboo_1/frame_0032_img.png +bamboo_1/frame_0033_img.png +bamboo_1/frame_0034_img.png +bamboo_1/frame_0035_img.png +bamboo_1/frame_0036_img.png +bamboo_1/frame_0037_img.png +bamboo_1/frame_0038_img.png +bamboo_1/frame_0039_img.png +bamboo_1/frame_0040_img.png +bamboo_1/frame_0041_img.png +bamboo_1/frame_0042_img.png +bamboo_1/frame_0043_img.png +bamboo_1/frame_0044_img.png +bamboo_1/frame_0045_img.png +bamboo_1/frame_0046_img.png +bamboo_1/frame_0047_img.png +bamboo_1/frame_0048_img.png +bamboo_1/frame_0049_img.png +bamboo_1/frame_0050_img.png +bamboo_2/frame_0001_img.png +bamboo_2/frame_0002_img.png +bamboo_2/frame_0003_img.png +bamboo_2/frame_0004_img.png +bamboo_2/frame_0005_img.png +bamboo_2/frame_0006_img.png +bamboo_2/frame_0007_img.png +bamboo_2/frame_0008_img.png +bamboo_2/frame_0009_img.png +bamboo_2/frame_0010_img.png +bamboo_2/frame_0011_img.png +bamboo_2/frame_0012_img.png +bamboo_2/frame_0013_img.png +bamboo_2/frame_0014_img.png +bamboo_2/frame_0015_img.png +bamboo_2/frame_0016_img.png +bamboo_2/frame_0017_img.png +bamboo_2/frame_0018_img.png +bamboo_2/frame_0019_img.png +bamboo_2/frame_0020_img.png +bamboo_2/frame_0021_img.png +bamboo_2/frame_0022_img.png +bamboo_2/frame_0023_img.png +bamboo_2/frame_0024_img.png +bamboo_2/frame_0025_img.png +bamboo_2/frame_0026_img.png +bamboo_2/frame_0027_img.png +bamboo_2/frame_0028_img.png +bamboo_2/frame_0029_img.png +bamboo_2/frame_0030_img.png +bamboo_2/frame_0031_img.png +bamboo_2/frame_0032_img.png +bamboo_2/frame_0033_img.png +bamboo_2/frame_0034_img.png +bamboo_2/frame_0035_img.png +bamboo_2/frame_0036_img.png +bamboo_2/frame_0037_img.png +bamboo_2/frame_0038_img.png +bamboo_2/frame_0039_img.png +bamboo_2/frame_0040_img.png +bamboo_2/frame_0041_img.png +bamboo_2/frame_0042_img.png +bamboo_2/frame_0043_img.png +bamboo_2/frame_0044_img.png +bamboo_2/frame_0045_img.png +bamboo_2/frame_0046_img.png +bamboo_2/frame_0047_img.png +bamboo_2/frame_0048_img.png +bamboo_2/frame_0049_img.png +bamboo_2/frame_0050_img.png +bandage_1/frame_0001_img.png +bandage_1/frame_0002_img.png +bandage_1/frame_0003_img.png +bandage_1/frame_0004_img.png +bandage_1/frame_0005_img.png +bandage_1/frame_0006_img.png +bandage_1/frame_0007_img.png +bandage_1/frame_0008_img.png +bandage_1/frame_0009_img.png +bandage_1/frame_0010_img.png +bandage_1/frame_0011_img.png +bandage_1/frame_0012_img.png +bandage_1/frame_0013_img.png +bandage_1/frame_0014_img.png +bandage_1/frame_0015_img.png +bandage_1/frame_0016_img.png +bandage_1/frame_0017_img.png +bandage_1/frame_0018_img.png +bandage_1/frame_0019_img.png +bandage_1/frame_0020_img.png +bandage_1/frame_0021_img.png +bandage_1/frame_0022_img.png +bandage_1/frame_0023_img.png +bandage_1/frame_0024_img.png +bandage_1/frame_0025_img.png +bandage_1/frame_0026_img.png +bandage_1/frame_0027_img.png +bandage_1/frame_0028_img.png +bandage_1/frame_0029_img.png +bandage_1/frame_0030_img.png +bandage_1/frame_0031_img.png +bandage_1/frame_0032_img.png +bandage_1/frame_0033_img.png +bandage_1/frame_0034_img.png +bandage_1/frame_0035_img.png +bandage_1/frame_0036_img.png +bandage_1/frame_0037_img.png +bandage_1/frame_0038_img.png +bandage_1/frame_0039_img.png +bandage_1/frame_0040_img.png +bandage_1/frame_0041_img.png +bandage_1/frame_0042_img.png +bandage_1/frame_0043_img.png +bandage_1/frame_0044_img.png +bandage_1/frame_0045_img.png +bandage_1/frame_0046_img.png +bandage_1/frame_0047_img.png +bandage_1/frame_0048_img.png +bandage_1/frame_0049_img.png +bandage_1/frame_0050_img.png +bandage_2/frame_0001_img.png +bandage_2/frame_0002_img.png +bandage_2/frame_0003_img.png +bandage_2/frame_0004_img.png +bandage_2/frame_0005_img.png +bandage_2/frame_0006_img.png +bandage_2/frame_0007_img.png +bandage_2/frame_0008_img.png +bandage_2/frame_0009_img.png +bandage_2/frame_0010_img.png +bandage_2/frame_0011_img.png +bandage_2/frame_0012_img.png +bandage_2/frame_0013_img.png +bandage_2/frame_0014_img.png +bandage_2/frame_0015_img.png +bandage_2/frame_0016_img.png +bandage_2/frame_0017_img.png +bandage_2/frame_0018_img.png +bandage_2/frame_0019_img.png +bandage_2/frame_0020_img.png +bandage_2/frame_0021_img.png +bandage_2/frame_0022_img.png +bandage_2/frame_0023_img.png +bandage_2/frame_0024_img.png +bandage_2/frame_0025_img.png +bandage_2/frame_0026_img.png +bandage_2/frame_0027_img.png +bandage_2/frame_0028_img.png +bandage_2/frame_0029_img.png +bandage_2/frame_0030_img.png +bandage_2/frame_0031_img.png +bandage_2/frame_0032_img.png +bandage_2/frame_0033_img.png +bandage_2/frame_0034_img.png +bandage_2/frame_0035_img.png +bandage_2/frame_0036_img.png +bandage_2/frame_0037_img.png +bandage_2/frame_0038_img.png +bandage_2/frame_0039_img.png +bandage_2/frame_0040_img.png +bandage_2/frame_0041_img.png +bandage_2/frame_0042_img.png +bandage_2/frame_0043_img.png +bandage_2/frame_0044_img.png +bandage_2/frame_0045_img.png +bandage_2/frame_0046_img.png +bandage_2/frame_0047_img.png +bandage_2/frame_0048_img.png +bandage_2/frame_0049_img.png +bandage_2/frame_0050_img.png +cave_2/frame_0001_img.png +cave_2/frame_0002_img.png +cave_2/frame_0003_img.png +cave_2/frame_0004_img.png +cave_2/frame_0005_img.png +cave_2/frame_0006_img.png +cave_2/frame_0007_img.png +cave_2/frame_0008_img.png +cave_2/frame_0009_img.png +cave_2/frame_0010_img.png +cave_2/frame_0011_img.png +cave_2/frame_0012_img.png +cave_2/frame_0013_img.png +cave_2/frame_0014_img.png +cave_2/frame_0015_img.png +cave_2/frame_0016_img.png +cave_2/frame_0017_img.png +cave_2/frame_0018_img.png +cave_2/frame_0019_img.png +cave_2/frame_0020_img.png +cave_2/frame_0021_img.png +cave_2/frame_0022_img.png +cave_2/frame_0023_img.png +cave_2/frame_0024_img.png +cave_2/frame_0025_img.png +cave_2/frame_0026_img.png +cave_2/frame_0027_img.png +cave_2/frame_0028_img.png +cave_2/frame_0029_img.png +cave_2/frame_0030_img.png +cave_2/frame_0031_img.png +cave_2/frame_0032_img.png +cave_2/frame_0033_img.png +cave_2/frame_0034_img.png +cave_2/frame_0035_img.png +cave_2/frame_0036_img.png +cave_2/frame_0037_img.png +cave_2/frame_0038_img.png +cave_2/frame_0039_img.png +cave_2/frame_0040_img.png +cave_2/frame_0041_img.png +cave_2/frame_0042_img.png +cave_2/frame_0043_img.png +cave_2/frame_0044_img.png +cave_2/frame_0045_img.png +cave_2/frame_0046_img.png +cave_2/frame_0047_img.png +cave_2/frame_0048_img.png +cave_2/frame_0049_img.png +cave_2/frame_0050_img.png +cave_4/frame_0001_img.png +cave_4/frame_0002_img.png +cave_4/frame_0003_img.png +cave_4/frame_0004_img.png +cave_4/frame_0005_img.png +cave_4/frame_0006_img.png +cave_4/frame_0007_img.png +cave_4/frame_0008_img.png +cave_4/frame_0009_img.png +cave_4/frame_0010_img.png +cave_4/frame_0011_img.png +cave_4/frame_0012_img.png +cave_4/frame_0013_img.png +cave_4/frame_0014_img.png +cave_4/frame_0015_img.png +cave_4/frame_0016_img.png +cave_4/frame_0017_img.png +cave_4/frame_0018_img.png +cave_4/frame_0019_img.png +cave_4/frame_0020_img.png +cave_4/frame_0021_img.png +cave_4/frame_0022_img.png +cave_4/frame_0023_img.png +cave_4/frame_0024_img.png +cave_4/frame_0025_img.png +cave_4/frame_0026_img.png +cave_4/frame_0027_img.png +cave_4/frame_0028_img.png +cave_4/frame_0029_img.png +cave_4/frame_0030_img.png +cave_4/frame_0031_img.png +cave_4/frame_0032_img.png +cave_4/frame_0033_img.png +cave_4/frame_0034_img.png +cave_4/frame_0035_img.png +cave_4/frame_0036_img.png +cave_4/frame_0037_img.png +cave_4/frame_0038_img.png +cave_4/frame_0039_img.png +cave_4/frame_0040_img.png +cave_4/frame_0041_img.png +cave_4/frame_0042_img.png +cave_4/frame_0043_img.png +cave_4/frame_0044_img.png +cave_4/frame_0045_img.png +cave_4/frame_0046_img.png +cave_4/frame_0047_img.png +cave_4/frame_0048_img.png +cave_4/frame_0049_img.png +cave_4/frame_0050_img.png +market_2/frame_0001_img.png +market_2/frame_0002_img.png +market_2/frame_0003_img.png +market_2/frame_0004_img.png +market_2/frame_0005_img.png +market_2/frame_0006_img.png +market_2/frame_0007_img.png +market_2/frame_0008_img.png +market_2/frame_0009_img.png +market_2/frame_0010_img.png +market_2/frame_0011_img.png +market_2/frame_0012_img.png +market_2/frame_0013_img.png +market_2/frame_0014_img.png +market_2/frame_0015_img.png +market_2/frame_0016_img.png +market_2/frame_0017_img.png +market_2/frame_0018_img.png +market_2/frame_0019_img.png +market_2/frame_0020_img.png +market_2/frame_0021_img.png +market_2/frame_0022_img.png +market_2/frame_0023_img.png +market_2/frame_0024_img.png +market_2/frame_0025_img.png +market_2/frame_0026_img.png +market_2/frame_0027_img.png +market_2/frame_0028_img.png +market_2/frame_0029_img.png +market_2/frame_0030_img.png +market_2/frame_0031_img.png +market_2/frame_0032_img.png +market_2/frame_0033_img.png +market_2/frame_0034_img.png +market_2/frame_0035_img.png +market_2/frame_0036_img.png +market_2/frame_0037_img.png +market_2/frame_0038_img.png +market_2/frame_0039_img.png +market_2/frame_0040_img.png +market_2/frame_0041_img.png +market_2/frame_0042_img.png +market_2/frame_0043_img.png +market_2/frame_0044_img.png +market_2/frame_0045_img.png +market_2/frame_0046_img.png +market_2/frame_0047_img.png +market_2/frame_0048_img.png +market_2/frame_0049_img.png +market_2/frame_0050_img.png +market_5/frame_0001_img.png +market_5/frame_0002_img.png +market_5/frame_0003_img.png +market_5/frame_0004_img.png +market_5/frame_0005_img.png +market_5/frame_0006_img.png +market_5/frame_0007_img.png +market_5/frame_0008_img.png +market_5/frame_0009_img.png +market_5/frame_0010_img.png +market_5/frame_0011_img.png +market_5/frame_0012_img.png +market_5/frame_0013_img.png +market_5/frame_0014_img.png +market_5/frame_0015_img.png +market_5/frame_0016_img.png +market_5/frame_0017_img.png +market_5/frame_0018_img.png +market_5/frame_0019_img.png +market_5/frame_0020_img.png +market_5/frame_0021_img.png +market_5/frame_0022_img.png +market_5/frame_0023_img.png +market_5/frame_0024_img.png +market_5/frame_0025_img.png +market_5/frame_0026_img.png +market_5/frame_0027_img.png +market_5/frame_0028_img.png +market_5/frame_0029_img.png +market_5/frame_0030_img.png +market_5/frame_0031_img.png +market_5/frame_0032_img.png +market_5/frame_0033_img.png +market_5/frame_0034_img.png +market_5/frame_0035_img.png +market_5/frame_0036_img.png +market_5/frame_0037_img.png +market_5/frame_0038_img.png +market_5/frame_0039_img.png +market_5/frame_0040_img.png +market_5/frame_0041_img.png +market_5/frame_0042_img.png +market_5/frame_0043_img.png +market_5/frame_0044_img.png +market_5/frame_0045_img.png +market_5/frame_0046_img.png +market_5/frame_0047_img.png +market_5/frame_0048_img.png +market_5/frame_0049_img.png +market_5/frame_0050_img.png +market_6/frame_0001_img.png +market_6/frame_0002_img.png +market_6/frame_0003_img.png +market_6/frame_0004_img.png +market_6/frame_0005_img.png +market_6/frame_0006_img.png +market_6/frame_0007_img.png +market_6/frame_0008_img.png +market_6/frame_0009_img.png +market_6/frame_0010_img.png +market_6/frame_0011_img.png +market_6/frame_0012_img.png +market_6/frame_0013_img.png +market_6/frame_0014_img.png +market_6/frame_0015_img.png +market_6/frame_0016_img.png +market_6/frame_0017_img.png +market_6/frame_0018_img.png +market_6/frame_0019_img.png +market_6/frame_0020_img.png +market_6/frame_0021_img.png +market_6/frame_0022_img.png +market_6/frame_0023_img.png +market_6/frame_0024_img.png +market_6/frame_0025_img.png +market_6/frame_0026_img.png +market_6/frame_0027_img.png +market_6/frame_0028_img.png +market_6/frame_0029_img.png +market_6/frame_0030_img.png +market_6/frame_0031_img.png +market_6/frame_0032_img.png +market_6/frame_0033_img.png +market_6/frame_0034_img.png +market_6/frame_0035_img.png +market_6/frame_0036_img.png +market_6/frame_0037_img.png +market_6/frame_0038_img.png +market_6/frame_0039_img.png +market_6/frame_0040_img.png +mountain_1/frame_0001_img.png +mountain_1/frame_0002_img.png +mountain_1/frame_0003_img.png +mountain_1/frame_0004_img.png +mountain_1/frame_0005_img.png +mountain_1/frame_0006_img.png +mountain_1/frame_0007_img.png +mountain_1/frame_0008_img.png +mountain_1/frame_0009_img.png +mountain_1/frame_0010_img.png +mountain_1/frame_0011_img.png +mountain_1/frame_0012_img.png +mountain_1/frame_0013_img.png +mountain_1/frame_0014_img.png +mountain_1/frame_0015_img.png +mountain_1/frame_0016_img.png +mountain_1/frame_0017_img.png +mountain_1/frame_0018_img.png +mountain_1/frame_0019_img.png +mountain_1/frame_0020_img.png +mountain_1/frame_0021_img.png +mountain_1/frame_0022_img.png +mountain_1/frame_0023_img.png +mountain_1/frame_0024_img.png +mountain_1/frame_0025_img.png +mountain_1/frame_0026_img.png +mountain_1/frame_0027_img.png +mountain_1/frame_0028_img.png +mountain_1/frame_0029_img.png +mountain_1/frame_0030_img.png +mountain_1/frame_0031_img.png +mountain_1/frame_0032_img.png +mountain_1/frame_0033_img.png +mountain_1/frame_0034_img.png +mountain_1/frame_0035_img.png +mountain_1/frame_0036_img.png +mountain_1/frame_0037_img.png +mountain_1/frame_0038_img.png +mountain_1/frame_0039_img.png +mountain_1/frame_0040_img.png +mountain_1/frame_0041_img.png +mountain_1/frame_0042_img.png +mountain_1/frame_0043_img.png +mountain_1/frame_0044_img.png +mountain_1/frame_0045_img.png +mountain_1/frame_0046_img.png +mountain_1/frame_0047_img.png +mountain_1/frame_0048_img.png +mountain_1/frame_0049_img.png +mountain_1/frame_0050_img.png +shaman_2/frame_0001_img.png +shaman_2/frame_0002_img.png +shaman_2/frame_0003_img.png +shaman_2/frame_0004_img.png +shaman_2/frame_0005_img.png +shaman_2/frame_0006_img.png +shaman_2/frame_0007_img.png +shaman_2/frame_0008_img.png +shaman_2/frame_0009_img.png +shaman_2/frame_0010_img.png +shaman_2/frame_0011_img.png +shaman_2/frame_0012_img.png +shaman_2/frame_0013_img.png +shaman_2/frame_0014_img.png +shaman_2/frame_0015_img.png +shaman_2/frame_0016_img.png +shaman_2/frame_0017_img.png +shaman_2/frame_0018_img.png +shaman_2/frame_0019_img.png +shaman_2/frame_0020_img.png +shaman_2/frame_0021_img.png +shaman_2/frame_0022_img.png +shaman_2/frame_0023_img.png +shaman_2/frame_0024_img.png +shaman_2/frame_0025_img.png +shaman_2/frame_0026_img.png +shaman_2/frame_0027_img.png +shaman_2/frame_0028_img.png +shaman_2/frame_0029_img.png +shaman_2/frame_0030_img.png +shaman_2/frame_0031_img.png +shaman_2/frame_0032_img.png +shaman_2/frame_0033_img.png +shaman_2/frame_0034_img.png +shaman_2/frame_0035_img.png +shaman_2/frame_0036_img.png +shaman_2/frame_0037_img.png +shaman_2/frame_0038_img.png +shaman_2/frame_0039_img.png +shaman_2/frame_0040_img.png +shaman_2/frame_0041_img.png +shaman_2/frame_0042_img.png +shaman_2/frame_0043_img.png +shaman_2/frame_0044_img.png +shaman_2/frame_0045_img.png +shaman_2/frame_0046_img.png +shaman_2/frame_0047_img.png +shaman_2/frame_0048_img.png +shaman_2/frame_0049_img.png +shaman_2/frame_0050_img.png +shaman_3/frame_0001_img.png +shaman_3/frame_0002_img.png +shaman_3/frame_0003_img.png +shaman_3/frame_0004_img.png +shaman_3/frame_0005_img.png +shaman_3/frame_0006_img.png +shaman_3/frame_0007_img.png +shaman_3/frame_0008_img.png +shaman_3/frame_0009_img.png +shaman_3/frame_0010_img.png +shaman_3/frame_0011_img.png +shaman_3/frame_0012_img.png +shaman_3/frame_0013_img.png +shaman_3/frame_0014_img.png +shaman_3/frame_0015_img.png +shaman_3/frame_0016_img.png +shaman_3/frame_0017_img.png +shaman_3/frame_0018_img.png +shaman_3/frame_0019_img.png +shaman_3/frame_0020_img.png +shaman_3/frame_0021_img.png +shaman_3/frame_0022_img.png +shaman_3/frame_0023_img.png +shaman_3/frame_0024_img.png +shaman_3/frame_0025_img.png +shaman_3/frame_0026_img.png +shaman_3/frame_0027_img.png +shaman_3/frame_0028_img.png +shaman_3/frame_0029_img.png +shaman_3/frame_0030_img.png +shaman_3/frame_0031_img.png +shaman_3/frame_0032_img.png +shaman_3/frame_0033_img.png +shaman_3/frame_0034_img.png +shaman_3/frame_0035_img.png +shaman_3/frame_0036_img.png +shaman_3/frame_0037_img.png +shaman_3/frame_0038_img.png +shaman_3/frame_0039_img.png +shaman_3/frame_0040_img.png +shaman_3/frame_0041_img.png +shaman_3/frame_0042_img.png +shaman_3/frame_0043_img.png +shaman_3/frame_0044_img.png +shaman_3/frame_0045_img.png +shaman_3/frame_0046_img.png +shaman_3/frame_0047_img.png +shaman_3/frame_0048_img.png +shaman_3/frame_0049_img.png +shaman_3/frame_0050_img.png +sleeping_1/frame_0001_img.png +sleeping_1/frame_0002_img.png +sleeping_1/frame_0003_img.png +sleeping_1/frame_0004_img.png +sleeping_1/frame_0005_img.png +sleeping_1/frame_0006_img.png +sleeping_1/frame_0007_img.png +sleeping_1/frame_0008_img.png +sleeping_1/frame_0009_img.png +sleeping_1/frame_0010_img.png +sleeping_1/frame_0011_img.png +sleeping_1/frame_0012_img.png +sleeping_1/frame_0013_img.png +sleeping_1/frame_0014_img.png +sleeping_1/frame_0015_img.png +sleeping_1/frame_0016_img.png +sleeping_1/frame_0017_img.png +sleeping_1/frame_0018_img.png +sleeping_1/frame_0019_img.png +sleeping_1/frame_0020_img.png +sleeping_1/frame_0021_img.png +sleeping_1/frame_0022_img.png +sleeping_1/frame_0023_img.png +sleeping_1/frame_0024_img.png +sleeping_1/frame_0025_img.png +sleeping_1/frame_0026_img.png +sleeping_1/frame_0027_img.png +sleeping_1/frame_0028_img.png +sleeping_1/frame_0029_img.png +sleeping_1/frame_0030_img.png +sleeping_1/frame_0031_img.png +sleeping_1/frame_0032_img.png +sleeping_1/frame_0033_img.png +sleeping_1/frame_0034_img.png +sleeping_1/frame_0035_img.png +sleeping_1/frame_0036_img.png +sleeping_1/frame_0037_img.png +sleeping_1/frame_0038_img.png +sleeping_1/frame_0039_img.png +sleeping_1/frame_0040_img.png +sleeping_1/frame_0041_img.png +sleeping_1/frame_0042_img.png +sleeping_1/frame_0043_img.png +sleeping_1/frame_0044_img.png +sleeping_1/frame_0045_img.png +sleeping_1/frame_0046_img.png +sleeping_1/frame_0047_img.png +sleeping_1/frame_0048_img.png +sleeping_1/frame_0049_img.png +sleeping_1/frame_0050_img.png +sleeping_2/frame_0001_img.png +sleeping_2/frame_0002_img.png +sleeping_2/frame_0003_img.png +sleeping_2/frame_0004_img.png +sleeping_2/frame_0005_img.png +sleeping_2/frame_0006_img.png +sleeping_2/frame_0007_img.png +sleeping_2/frame_0008_img.png +sleeping_2/frame_0009_img.png +sleeping_2/frame_0010_img.png +sleeping_2/frame_0011_img.png +sleeping_2/frame_0012_img.png +sleeping_2/frame_0013_img.png +sleeping_2/frame_0014_img.png +sleeping_2/frame_0015_img.png +sleeping_2/frame_0016_img.png +sleeping_2/frame_0017_img.png +sleeping_2/frame_0018_img.png +sleeping_2/frame_0019_img.png +sleeping_2/frame_0020_img.png +sleeping_2/frame_0021_img.png +sleeping_2/frame_0022_img.png +sleeping_2/frame_0023_img.png +sleeping_2/frame_0024_img.png +sleeping_2/frame_0025_img.png +sleeping_2/frame_0026_img.png +sleeping_2/frame_0027_img.png +sleeping_2/frame_0028_img.png +sleeping_2/frame_0029_img.png +sleeping_2/frame_0030_img.png +sleeping_2/frame_0031_img.png +sleeping_2/frame_0032_img.png +sleeping_2/frame_0033_img.png +sleeping_2/frame_0034_img.png +sleeping_2/frame_0035_img.png +sleeping_2/frame_0036_img.png +sleeping_2/frame_0037_img.png +sleeping_2/frame_0038_img.png +sleeping_2/frame_0039_img.png +sleeping_2/frame_0040_img.png +sleeping_2/frame_0041_img.png +sleeping_2/frame_0042_img.png +sleeping_2/frame_0043_img.png +sleeping_2/frame_0044_img.png +sleeping_2/frame_0045_img.png +sleeping_2/frame_0046_img.png +sleeping_2/frame_0047_img.png +sleeping_2/frame_0048_img.png +sleeping_2/frame_0049_img.png +sleeping_2/frame_0050_img.png +temple_2/frame_0001_img.png +temple_2/frame_0002_img.png +temple_2/frame_0003_img.png +temple_2/frame_0004_img.png +temple_2/frame_0005_img.png +temple_2/frame_0006_img.png +temple_2/frame_0007_img.png +temple_2/frame_0008_img.png +temple_2/frame_0009_img.png +temple_2/frame_0010_img.png +temple_2/frame_0011_img.png +temple_2/frame_0012_img.png +temple_2/frame_0013_img.png +temple_2/frame_0014_img.png +temple_2/frame_0015_img.png +temple_2/frame_0016_img.png +temple_2/frame_0017_img.png +temple_2/frame_0018_img.png +temple_2/frame_0019_img.png +temple_2/frame_0020_img.png +temple_2/frame_0021_img.png +temple_2/frame_0022_img.png +temple_2/frame_0023_img.png +temple_2/frame_0024_img.png +temple_2/frame_0025_img.png +temple_2/frame_0026_img.png +temple_2/frame_0027_img.png +temple_2/frame_0028_img.png +temple_2/frame_0029_img.png +temple_2/frame_0030_img.png +temple_2/frame_0031_img.png +temple_2/frame_0032_img.png +temple_2/frame_0033_img.png +temple_2/frame_0034_img.png +temple_2/frame_0035_img.png +temple_2/frame_0036_img.png +temple_2/frame_0037_img.png +temple_2/frame_0038_img.png +temple_2/frame_0039_img.png +temple_2/frame_0040_img.png +temple_2/frame_0041_img.png +temple_2/frame_0042_img.png +temple_2/frame_0043_img.png +temple_2/frame_0044_img.png +temple_2/frame_0045_img.png +temple_2/frame_0046_img.png +temple_2/frame_0047_img.png +temple_2/frame_0048_img.png +temple_2/frame_0049_img.png +temple_2/frame_0050_img.png +temple_3/frame_0001_img.png +temple_3/frame_0002_img.png +temple_3/frame_0003_img.png +temple_3/frame_0004_img.png +temple_3/frame_0005_img.png +temple_3/frame_0006_img.png +temple_3/frame_0007_img.png +temple_3/frame_0008_img.png +temple_3/frame_0009_img.png +temple_3/frame_0010_img.png +temple_3/frame_0011_img.png +temple_3/frame_0012_img.png +temple_3/frame_0013_img.png +temple_3/frame_0014_img.png +temple_3/frame_0015_img.png +temple_3/frame_0016_img.png +temple_3/frame_0017_img.png +temple_3/frame_0018_img.png +temple_3/frame_0019_img.png +temple_3/frame_0020_img.png +temple_3/frame_0021_img.png +temple_3/frame_0022_img.png +temple_3/frame_0023_img.png +temple_3/frame_0024_img.png +temple_3/frame_0025_img.png +temple_3/frame_0026_img.png +temple_3/frame_0027_img.png +temple_3/frame_0028_img.png +temple_3/frame_0029_img.png +temple_3/frame_0030_img.png +temple_3/frame_0031_img.png +temple_3/frame_0032_img.png +temple_3/frame_0033_img.png +temple_3/frame_0034_img.png +temple_3/frame_0035_img.png +temple_3/frame_0036_img.png +temple_3/frame_0037_img.png +temple_3/frame_0038_img.png +temple_3/frame_0039_img.png +temple_3/frame_0040_img.png +temple_3/frame_0041_img.png +temple_3/frame_0042_img.png +temple_3/frame_0043_img.png +temple_3/frame_0044_img.png +temple_3/frame_0045_img.png +temple_3/frame_0046_img.png +temple_3/frame_0047_img.png +temple_3/frame_0048_img.png +temple_3/frame_0049_img.png +temple_3/frame_0050_img.png \ No newline at end of file diff --git a/infer/image_utils.py b/infer/image_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4721fabf708537e8dc40c0a89bb8e76a2067cd3b --- /dev/null +++ b/infer/image_utils.py @@ -0,0 +1,158 @@ +from PIL import Image +import matplotlib +import numpy as np + +from typing import List, Union +import PIL.Image + +import torch +from torchvision.transforms import InterpolationMode +from torchvision.transforms.functional import resize + +def concatenate_images(*image_lists): + # Ensure at least one image list is provided + if not image_lists or not image_lists[0]: + raise ValueError("At least one non-empty image list must be provided") + + # Determine the maximum width of any single row and the total height + max_width = 0 + total_height = 0 + row_widths = [] + row_heights = [] + + # Compute dimensions for each row + for image_list in image_lists: + if image_list: # Ensure the list is not empty + width = sum(img.width for img in image_list) + height = image_list[0].height # Assuming all images in the list have the same height + max_width = max(max_width, width) + total_height += height + row_widths.append(width) + row_heights.append(height) + + # Create a new image to concatenate everything into + new_image = Image.new('RGB', (max_width, total_height)) + + # Concatenate each row of images + y_offset = 0 + for i, image_list in enumerate(image_lists): + x_offset = 0 + for img in image_list: + new_image.paste(img, (x_offset, y_offset)) + x_offset += img.width + y_offset += row_heights[i] # Move the offset down to the next row + + return new_image + + +def colorize_depth_map(depth, mask=None, reverse_color=False): + cm = matplotlib.colormaps["Spectral"] + # normalize + depth = ((depth - depth.min()) / (depth.max() - depth.min())) + # colorize + if reverse_color: + img_colored_np = cm(1 - depth, bytes=False)[:, :, 0:3] # Invert the depth values before applying colormap + else: + img_colored_np = cm(depth, bytes=False)[:, :, 0:3] # (h,w,3) + + depth_colored = (img_colored_np * 255).astype(np.uint8) + if mask is not None: + masked_image = np.zeros_like(depth_colored) + masked_image[mask.numpy()] = depth_colored[mask.numpy()] + depth_colored_img = Image.fromarray(masked_image) + else: + depth_colored_img = Image.fromarray(depth_colored) + return depth_colored_img + + +def resize_max_res( + img: torch.Tensor, + max_edge_resolution: int, + resample_method: InterpolationMode = InterpolationMode.BILINEAR, +) -> torch.Tensor: + """ + Resize image to limit maximum edge length while keeping aspect ratio. + + Args: + img (`torch.Tensor`): + Image tensor to be resized. Expected shape: [B, C, H, W] + max_edge_resolution (`int`): + Maximum edge length (pixel). + resample_method (`PIL.Image.Resampling`): + Resampling method used to resize images. + + Returns: + `torch.Tensor`: Resized image. + """ + assert 4 == img.dim(), f"Invalid input shape {img.shape}" + + original_height, original_width = img.shape[-2:] + downscale_factor = min( + max_edge_resolution / original_width, max_edge_resolution / original_height + ) + + new_width = int(original_width * downscale_factor) + new_height = int(original_height * downscale_factor) + + resized_img = resize(img, (new_height, new_width), resample_method, antialias=True) + return resized_img + +def resize_back( + img: Union[torch.Tensor, np.ndarray, PIL.Image.Image, List[PIL.Image.Image]], + target_size: Union[int, tuple[int, int]], + resample_method: Union[InterpolationMode, int] = InterpolationMode.BILINEAR, +) -> Union[torch.Tensor, np.ndarray, PIL.Image.Image, List[PIL.Image.Image]]: + """ + Resize image to target size. + + Args: + img (`Union[torch.Tensor, np.ndarray, PIL.Image.Image, List[PIL.Image.Image]]`): + Image to be resized. + target_size (`Union[int, tuple[int, int]]`): + Target size of the resized image. + resample_method (`Union[InterpolationMode, int]`): + Resampling method used to resize images. + + Returns: + `Union[torch.Tensor, np.ndarray, PIL.Image.Image, List[PIL.Image.Image]]`: Resized image. + """ + if isinstance(img, torch.Tensor): # [B, C, H, W] + resized_img = resize(img, target_size, resample_method, antialias=True) + if isinstance(img, np.ndarray): # [B, H, W, C] + # Conver to Torch.Tensor + img = torch.tensor(img).permute(0, 3, 1, 2) + resized_img = resize(img, target_size, resample_method, antialias=True) + # Convert back to np.ndarray + resized_img = resized_img.permute(0, 2, 3, 1).numpy() + elif isinstance(img, PIL.Image.Image): + target_size = (target_size[1], target_size[0]) # PIL uses (width, height) + resized_img = img.resize(target_size, resample_method) + elif isinstance(img, list) and all(isinstance(i, PIL.Image.Image) for i in img): + target_size = (target_size[1], target_size[0]) # PIL uses (width, height) + resized_img = [i.resize(target_size, resample_method) for i in img] + return resized_img + +def get_pil_resample_method(method_str: str) -> int: + resample_method_dict = { + "bilinear": Image.BILINEAR, + "bicubic": Image.BICUBIC, + "nearest": Image.NEAREST, + } + resample_method = resample_method_dict.get(method_str, None) + if resample_method is None: + raise ValueError(f"Unknown resampling method: {resample_method}") + else: + return resample_method + +def get_tv_resample_method(method_str: str) -> InterpolationMode: + resample_method_dict = { + "bilinear": InterpolationMode.BILINEAR, + "bicubic": InterpolationMode.BICUBIC, + "nearest": InterpolationMode.NEAREST_EXACT, + "nearest-exact": InterpolationMode.NEAREST_EXACT, + } + resample_method = resample_method_dict.get(method_str, None) + if resample_method is None: + raise ValueError(f"Unknown resampling method: {resample_method}") + else: + return resample_method diff --git a/infer/inference.py b/infer/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..5802ddd612ebb582425f0cc09b120add6bb47c06 --- /dev/null +++ b/infer/inference.py @@ -0,0 +1,569 @@ +import argparse +import json +import itertools +import math +import os +import sys +import time +from pathlib import Path + +# 添加父目录到系统路径 +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import numpy as np +import torch +from einops import rearrange, repeat +from PIL import Image +from safetensors.torch import load_file +from torchvision.transforms import functional as F +from tqdm import tqdm +import torch.nn.functional as Func + +import infer.sampling as sampling +from modules.autoencoder import AutoEncoder +from modules.model_edit import Step1XParams, Step1XEdit + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_QWEN_DIR = REPO_ROOT / "Qwen" +EMPTY_PROMPT_LATENT_PATH = REPO_ROOT / "latent" / "no_info.npz" + + +def cudagc(): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + +def load_state_dict(model, ckpt_path, device="cuda", strict=False, assign=True): + if Path(ckpt_path).suffix == ".safetensors": + state_dict = load_file(ckpt_path, device) + else: + state_dict = torch.load(ckpt_path, map_location="cpu") + + missing, unexpected = model.load_state_dict(state_dict, strict=strict, assign=assign) + if len(missing) > 0 and len(unexpected) > 0: + print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing)) + print("\n" + "-" * 79 + "\n") + print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected)) + elif len(missing) > 0: + print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing)) + elif len(unexpected) > 0: + print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected)) + return model + + +def load_models(dit_path=None, ae_path=None, qwen2vl_model_path=None, device="cuda", max_length=256, dtype=torch.bfloat16, args=None): + empty_llm = args is not None and hasattr(args, 'prompt_type') and args.prompt_type == 'empty' + if empty_llm: + print("[INFO] prompt_type=empty, 跳过Qwen模型加载") + qwen2vl_encoder = None + else: + # Lazy import to avoid pulling transformers/vision stack during evaluation with prompt_type=empty. + from modules.conditioner import Qwen25VL_7b_Embedder as Qwen2VLEmbedder + qwen2vl_encoder = Qwen2VLEmbedder( + qwen2vl_model_path, + device=device, + max_length=max_length, + dtype=dtype, + args=args, + ) + + with torch.device("meta"): + ae = AutoEncoder( + resolution=256, + in_channels=3, + ch=128, + out_ch=3, + ch_mult=[1, 2, 4, 4], + num_res_blocks=2, + z_channels=16, + scale_factor=0.3611, + shift_factor=0.1159, + ) + + step1x_params = Step1XParams( + in_channels=64, + out_channels=64, + vec_in_dim=768, + context_in_dim=4096, + hidden_size=3072, + mlp_ratio=4.0, + num_heads=24, + depth=19, + depth_single_blocks=38, + axes_dim=[16, 56, 56], + theta=10_000, + qkv_bias=True, + ) + dit = Step1XEdit(step1x_params) + + ae = load_state_dict(ae, ae_path, 'cpu') + dit = load_state_dict(dit, dit_path, 'cpu') + + ae = ae.to(dtype=torch.float32) + + return ae, dit, qwen2vl_encoder + + +def equip_dit_with_lora_sd_scripts(ae, text_encoders, dit, lora, device='cuda'): + from safetensors.torch import load_file + weights_sd = load_file(lora) + is_lora = True + from library import lora_module + module = lora_module + lora_model, _ = module.create_network_from_weights(1.0, None, ae, text_encoders, dit, weights_sd, True) + lora_model.merge_to(text_encoders, dit, weights_sd) + + lora_model.set_multiplier(1.0) + return lora_model + +class ImageGenerator: + + def __init__( + self, + dit_path=None, + ae_path=None, + qwen2vl_model_path=None, + device="cuda", + max_length=640, + dtype=torch.bfloat16, + quantized=False, + offload=False, + lora=None, + args=None, + ) -> None: + self.device = torch.device(device) + self.args = args + self.ae, self.dit, self.llm_encoder = load_models( + dit_path=dit_path, + ae_path=ae_path, + qwen2vl_model_path=qwen2vl_model_path, + max_length=max_length, + dtype=dtype, + device=device, + args=args, + ) + if not quantized: + self.dit = self.dit.to(dtype=torch.bfloat16) + else: + self.dit = self.dit.to(dtype=torch.float8_e4m3fn) + if not offload: + self.dit = self.dit.to(device=self.device) + self.ae = self.ae.to(device=self.device) + self.quantized = quantized + self.offload = offload + if lora is not None: + self.lora_module = equip_dit_with_lora_sd_scripts( + self.ae, + [self.llm_encoder], + self.dit, + lora, + device=self.dit.device, + ) + else: + self.lora_module = None + + def prepare(self, prompt, img, ref_image, ref_image_raw, empty_llm=False): + bs, _, h, w = img.shape + bs, _, ref_h, ref_w = ref_image.shape + + assert h == ref_h and w == ref_w + + if bs == 1 and not isinstance(prompt, str): + bs = len(prompt) + elif bs >= 1 and isinstance(prompt, str): + prompt = [prompt] * bs + + img = rearrange(img, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=2, pw=2) #2,16,82,110->2,2255,64 + ref_img = rearrange(ref_image, "b c (ref_h ph) (ref_w pw) -> b (ref_h ref_w) (c ph pw)", ph=2, pw=2) # 将二维图像"压平"成一维序列 这是为 Transformer 模型准备的，因为它处理的是序列数据 + if img.shape[0] == 1 and bs > 1: + img = repeat(img, "1 ... -> bs ...", bs=bs) + ref_img = repeat(ref_img, "1 ... -> bs ...", bs=bs) + #img 和 ref_img 已经不再是二维的图片了，而是变成了一个 "patches" (图像块) 的序列。一个块是64维度的。Transformer不知道这2255个图像块哪个在左上角，哪个在右下角。 + img_ids = torch.zeros(h // 2, w // 2, 3) #41,55,3 # h 和 w 是潜在空间的高和宽，但 rearrange 操作把 2x2 的小块合并了# 所以实际的网格大小是 h/2 x w/2# 最后的 3 代表每个坐标有3个分量 (一个预留, Y坐标, X坐标) + img_ids[..., 1] = img_ids[..., 1] + torch.arange(h // 2)[:, None] + img_ids[..., 2] = img_ids[..., 2] + torch.arange(w // 2)[None, :] #通过广播机制，第 i 行的所有点的第二个分量都被赋值为 i + img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs) #将二维坐标网格"压平"成一维序列，并复制到对应的batch size + + ref_img_ids = torch.zeros(ref_h // 2, ref_w // 2, 3) + + ref_img_ids[..., 1] = ref_img_ids[..., 1] + torch.arange(ref_h // 2)[:, None] + ref_img_ids[..., 2] = ref_img_ids[..., 2] + torch.arange(ref_w // 2)[None, :] + ref_img_ids = repeat(ref_img_ids, "ref_h ref_w c -> b (ref_h ref_w) c", b=bs) + + if isinstance(prompt, str): + prompt = [prompt] + + if self.offload: + self.llm_encoder = self.llm_encoder.to(self.device) + + if empty_llm: + empty_prompt_cache = getattr(self.args, "empty_prompt_cache", None) if self.args is not None else None + cache_path = Path(empty_prompt_cache) if empty_prompt_cache else EMPTY_PROMPT_LATENT_PATH + data = np.load(cache_path) + txt = torch.from_numpy(data['embeds']).to(img.device).unsqueeze(0) + txt = torch.cat([txt, txt], dim=0) + mask = torch.from_numpy(data['masks']).to(img.device).unsqueeze(0) + mask = torch.cat([mask, mask], dim=0) + else: + txt, mask = self.llm_encoder(prompt, ref_image_raw) #之所以都要复制一份，是因为有正负两种prompt + + if self.offload: + self.llm_encoder = self.llm_encoder.cpu() + cudagc() + + txt_ids = torch.zeros(bs, txt.shape[1], 3) + + img = torch.cat([img, ref_img.to(device=img.device, dtype=img.dtype)], dim=-2) #2,4550,64 在patch上concat？？？ + img_ids = torch.cat([img_ids, ref_img_ids], dim=-2) + + return { + "img": img, + "mask": mask, + "img_ids": img_ids.to(img.device), #图像坐标 + "llm_embedding": txt.to(img.device), #文字向量 + "txt_ids": txt_ids.to(img.device), #文字坐标 + } + + @staticmethod + def process_diff_norm(diff_norm, k): + pow_result = torch.pow(diff_norm, k) + + result = torch.where( + diff_norm > 1.0, + pow_result, + torch.where(diff_norm < 1.0, torch.ones_like(diff_norm), diff_norm), + ) + return result + + def denoise( + self, + img: torch.Tensor, + img_ids: torch.Tensor, + llm_embedding: torch.Tensor, + txt_ids: torch.Tensor, + timesteps: list[float], + cfg_guidance: float = 6.0, + mask=None, + show_progress=False, + timesteps_truncate=1.0, + ): + if self.offload: + self.dit = self.dit.to(self.device) + if show_progress: + pbar = tqdm(itertools.pairwise(timesteps), desc='denoising...') + else: + pbar = itertools.pairwise(timesteps) + ''' + Cond 0 RGB + Uncd 0 RGB + ''' + for t_curr, t_prev in pbar: + ''' + 若输入维度是2，无所谓，维度是1则： + imgN D RGB + imgN D RGB + ''' + if img.shape[0] == 1 and cfg_guidance != -1: + img = torch.cat([img, img], dim=0) + t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device) + + pred, feat = self.dit( + img=img, + img_ids=img_ids, + txt_ids=txt_ids, + timesteps=t_vec, + llm_embedding=llm_embedding, + t_vec=t_vec, + mask=mask, + ) + + assert cfg_guidance != -1, " cfg_guidance must not be -1 NOW!!!!" + cond, uncond = ( + pred[0:pred.shape[0] // 2, :], + pred[pred.shape[0] // 2:, :], + ) + ''' + Cond D ??? <- pred + Uncd D ??? + ''' + pred = uncond + cfg_guidance * (cond - uncond) #1,4608,64 + pred1 = cond #todo only support single denoise!!! + ''' + Cond 0 RGB + + pred D ??? + temI D ??? + ''' + tem_img = img[0:img.shape[0] // 2, :] + (t_prev - t_curr) * pred #1,4608,64 + img_input_length = img.shape[1] // 2 + ''' + tmpI [D](√) ???(x) + cat Cond 0(x) [RGB](√) + imgN [D] [RGB] + ''' + img = torch.cat( + [ + tem_img[:, :img_input_length], #1,2304,64 + img[:img.shape[0] // 2, img_input_length:], #1,2304,64 + ], + dim=1) #1,4608,64 + if self.offload: + self.dit = self.dit.cpu() + cudagc() + + return img[:, :img.shape[1] // 2], pred1[:, img.shape[1] // 2:] + + def double_denoise(self,img,img_ids,llm_embedding,txt_ids,timesteps,cfg_guidance=6.0,mask=None,height=None,width=None): + if img.shape[0] == 1 and cfg_guidance != -1: + img = torch.cat([img, img], dim=0) + + t_vec = torch.full((img.shape[0],), 1.0, dtype=img.dtype, device=img.device) + + pred, _ = self.dit( + img=img, + img_ids=img_ids, + txt_ids=txt_ids, + timesteps=t_vec, + llm_embedding=llm_embedding, + t_vec=t_vec, + mask=mask, + ) + + assert cfg_guidance != -1, " cfg_guidance must not be -1 NOW!!!!" + pred, uncond = ( + pred[0:pred.shape[0] // 2, :], + pred[pred.shape[0] // 2:, :], + ) + Lpred,Rpred = self.unpack_latents(pred, height//16, width//16) + return Lpred.to(torch.float32),Rpred.to(torch.float32) + + @staticmethod + def unpack(x: torch.Tensor, height: int, width: int) -> torch.Tensor: + return rearrange( + x, + "b (h w) (c ph pw) -> b c (h ph) (w pw)", + h=math.ceil(height / 16), + w=math.ceil(width / 16), + ph=2, + pw=2, + ) + + @staticmethod + def unpack_latents(x: torch.Tensor, packed_latent_height: int, packed_latent_width: int): + """ + x: [b (h w) (c ph pw)] -> [b c (h ph) (w pw)], ph=2, pw=2 + """ + import einops + x = einops.rearrange(x, "b (p h w) (c ph pw) -> b p c (h ph) (w pw)", h=packed_latent_height, w=packed_latent_width, ph=2, pw=2, p=2) + return x[:, 0], x[:, 1] + + @staticmethod + def load_image(image): + from PIL import Image + + if isinstance(image, np.ndarray): + image = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0 + image = image.unsqueeze(0) + return image + elif isinstance(image, Image.Image): + image = F.to_tensor(image.convert("RGB")) + image = image.unsqueeze(0) + return image + elif isinstance(image, torch.Tensor): + return image + elif isinstance(image, str): + image = F.to_tensor(Image.open(image).convert("RGB")) + image = image.unsqueeze(0) + return image + else: + raise ValueError(f"Unsupported image type: {type(image)}") + + def output_process_image(self, resize_img, image_size): + res_image = resize_img.resize(image_size) + return res_image + + def input_process_image(self, img): + if isinstance(img, torch.Tensor): + w, h = img.shape[-1], img.shape[-2] + elif isinstance(img, Image.Image): + w, h = img.size + + if w <= 1024 and h <= 768: + w_new, h_new = 1024, 768 + elif w <= 1280 and h <= 960: + w_new, h_new = 1216, 352 + elif w <= 6048 and h <= 4032: + w_new, h_new = 864, 576 + else: + w_new, h_new = w, h + + if isinstance(img, torch.Tensor): + img_resized = Func.interpolate(img, (h_new, w_new), mode='bilinear', align_corners=False) + img_resized = img_resized.clamp(0, 1) + else: + img_resized = img.resize((w_new, h_new)) + + return img_resized, (w_new, h_new) + + @torch.inference_mode() + def generate_image( + self,prompt,negative_prompt,ref_images,num_steps,cfg_guidance,seed,num_samples=1,init_image=None,image2image_strength=0.0,show_progress=False,size_level=512,args=None,judge=None,name=None + ): + assert num_samples == 1, "num_samples > 1 is not supported yet." + + ref_images_raw, img_info = self.input_process_image(ref_images) + if isinstance(ref_images, Image.Image): + ref_images_raw = self.load_image(ref_images_raw) + + height, width = ref_images_raw.shape[-2], ref_images_raw.shape[-1] + + ref_images_raw = ref_images_raw.to(self.device) + if self.offload: + self.ae = self.ae.to(self.device) + ref_images = self.ae.encode(ref_images_raw.to(self.device) * 2 - 1) #bs,3,656,880 -> 1,16,82,110 + #加入cache + + if self.offload: + self.ae = self.ae.cpu() + cudagc() + + seed = int(seed) + seed = torch.Generator(device="cpu").seed() if seed < 0 else seed + + t0 = time.perf_counter() + + if init_image is not None: + init_image = self.load_image(init_image) + init_image = init_image.to(self.device) + init_image = torch.nn.functional.interpolate(init_image, (height, width)) + if self.offload: + self.ae = self.ae.to(self.device) + init_image = self.ae.encode(init_image.to() * 2 - 1) + if self.offload: + self.ae = self.ae.cpu() + cudagc() + + if args is not None and hasattr(args, 'single_denoise') and not args.single_denoise: + x = torch.randn(num_samples,16,height // 8,width // 8,device=self.device,dtype=torch.bfloat16,generator=torch.Generator(device=self.device).manual_seed(seed),) + else: + x= torch.zeros(num_samples,16,height // 8,width // 8,device=self.device,dtype=torch.bfloat16,) + + + timesteps = sampling.get_schedule(num_steps, x.shape[-1] * x.shape[-2] // 4, shift=True) + + if init_image is not None: + t_idx = int((1 - image2image_strength) * num_steps) + t = timesteps[t_idx] + timesteps = timesteps[t_idx:] + x = t * x + (1.0 - t) * init_image.to(x.dtype) + + x = torch.cat([x, x], dim=0) + ref_images = torch.cat([ref_images, ref_images], dim=0) #这里是为了有无prompt + ref_images_raw = torch.cat([ref_images_raw, ref_images_raw], dim=0) + + # 检查args和prompt_type属性 + empty_llm = args is not None and hasattr(args, 'prompt_type') and args.prompt_type == 'empty' + + inputs = self.prepare( + [prompt, negative_prompt], + x, #img这个gt给的是全噪声在推理 + ref_image=ref_images, + ref_image_raw=ref_images_raw, + empty_llm=empty_llm) + + with torch.autocast(device_type=self.device.type,dtype=torch.bfloat16): + # Lpred,Rpred = self.double_denoise(**inputs,cfg_guidance=cfg_guidance,timesteps=timesteps,height=height,width=width)#图像中包括ref image + Lpred,Rpred = self.denoise(**inputs,cfg_guidance=cfg_guidance,timesteps=timesteps,show_progress=show_progress,timesteps_truncate=1.0,)#图像中包括ref image + Lpred=self.unpack(Lpred.float(),height,width) + Rpred=self.unpack(Rpred.float(),height,width) + if judge is not None: + judge = Func.interpolate(judge, (height, width), mode='bilinear', align_corners=False) + training_gt=self.ae.encode(judge) + traing_loss = torch.nn.functional.mse_loss(Rpred,training_gt) + print(f"training_loss with rgb2: {traing_loss}") + + norm = torch.linalg.norm(judge, dim=1, keepdim=True) + norm[norm < 1e-9] = 1e-9 + judge = judge / norm + training_gt =self.ae.encode(judge) + training_loss = torch.nn.functional.mse_loss(Rpred,training_gt) + print(f"training_loss with normed_rgb: {training_loss}") + Lpred = self.ae.decode(Lpred) + Rpred = self.ae.decode(Rpred) + + + Lpred = Lpred.clamp(-1, 1) + Lpred = Lpred.mul(0.5).add(0.5) + Rpred = Rpred.clamp(-1, 1) + # Rpred = Rpred.mul(0.5).add(0.5) + + images_list = [] + for img in Rpred.float(): + images_list.append(self.output_process_image(F.to_pil_image(img), img_info)) + return images_list, Lpred.float(), Rpred.float() + + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('--model_path', type=str, required=True, help='Path to the model checkpoint') + parser.add_argument('--input_dir', type=str, required=True, help='Path to the input image directory') + parser.add_argument('--output_dir', type=str, required=True, help='Path to the output image directory') + parser.add_argument('--json_path', type=str, required=True, help='Path to the JSON file containing image names and prompts') + parser.add_argument('--seed', type=int, default=42, help='Random seed for generation') + parser.add_argument('--num_steps', type=int, default=28, help='Number of diffusion steps') + parser.add_argument('--cfg_guidance', type=float, default=6.0, help='CFG guidance strength') + parser.add_argument('--size_level', default=512, type=int) + parser.add_argument('--offload', action='store_true', help='Use offload for large models') + parser.add_argument('--quantized', action='store_true', help='Use fp8 model weights') + parser.add_argument('--lora', type=str, default=None) + parser.add_argument('--qwen2vl_model_path', type=str, default=str(DEFAULT_QWEN_DIR), help='Path to the local Qwen2.5-VL model directory') + parser.add_argument('--empty_prompt_cache', type=str, default=str(EMPTY_PROMPT_LATENT_PATH), help='Path to the empty-prompt latent cache') + args = parser.parse_args() + + assert os.path.exists(args.input_dir), f"Input directory {args.input_dir} does not exist." + assert os.path.exists(args.json_path), f"JSON file {args.json_path} does not exist." + + args.output_dir = args.output_dir.rstrip('/') + ('-offload' if args.offload else "") + ('-quantized' if args.quantized else "") + f"-{args.size_level}" + os.makedirs(args.output_dir, exist_ok=True) + + image_and_prompts = json.load(open(args.json_path, 'r')) + + image_edit = ImageGenerator( + ae_path=os.path.join(args.model_path, 'vae.safetensors'), + dit_path=os.path.join(args.model_path, "step1x-edit-i1258-FP8.safetensors" if args.quantized else "step1x-edit-i1258.safetensors"), + qwen2vl_model_path=args.qwen2vl_model_path, + max_length=640, + quantized=args.quantized, + offload=args.offload, + lora=args.lora, + ) + + time_list = [] + for image_name, prompt in image_and_prompts.items(): + image_path = os.path.join(args.input_dir, image_name) + output_path = os.path.join(args.output_dir, image_name) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + start_time = time.time() + + images, _, _ = image_edit.generate_image( + prompt, + negative_prompt="", + ref_images=Image.open(image_path).convert("RGB"), + num_samples=1, + num_steps=args.num_steps, + cfg_guidance=args.cfg_guidance, + seed=args.seed, + show_progress=True, + size_level=args.size_level, + ) + + print(f"Time taken: {time.time() - start_time:.2f} seconds") + time_list.append(time.time() - start_time) + + images[0].save(output_path, lossless=True) + if len(time_list) > 1: + print(f'average time for {args.output_dir}: ', sum(time_list[1:]) / len(time_list[1:])) + + +if __name__ == "__main__": + main() diff --git a/infer/inner_evaluation.py b/infer/inner_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..2dd2d79cccdb316da88c76b7c8b6f5029c74396b --- /dev/null +++ b/infer/inner_evaluation.py @@ -0,0 +1,598 @@ +import logging +import os +import sys +import csv # 保留csv库用于保存结果 +import multiprocessing as mp +import time +import numpy as np +import torch +from omegaconf import OmegaConf +from tabulate import tabulate +from torch.utils.data import DataLoader +from tqdm.auto import tqdm +import cv2 +from infer.dataset import ( + BaseDepthDataset, + DatasetMode, + get_dataset, + get_pred_name, +) +from .util import metric, normal_utils +from .util.alignment import (align_depth_least_square, depth2disparity, disparity2depth, depth2log_space, log_space2depth) +from .util.metric import MetricTracker +from infer.image_utils import colorize_depth_map + +eval_metrics = [ + "abs_relative_difference", + "squared_relative_difference", + "rmse_linear", + "rmse_log", + "delta1_acc", + "delta2_acc", + "delta3_acc", +] + + +def save_visualization_worker(save_vis_path, safe_pred_name, cfg_suffix, depth_pred_np, depth_raw_np, valid_mask_np, input_rgb_data, rank): + """ + 可视化保存的工作函数，在独立进程中运行 + Args: + save_vis_path: 保存路径 + safe_pred_name: 安全的预测文件名 + cfg_suffix: cfg后缀 + depth_pred_np: 预测深度图 numpy数组 + depth_raw_np: GT深度图 numpy数组 + valid_mask_np: 有效掩码 numpy数组 + input_rgb_data: 输入RGB图像数据 + rank: GPU rank + """ + try: + # 转换为torch tensor用于colorize_depth_map + depth_pred_ts = torch.from_numpy(depth_pred_np) + depth_raw_ts = torch.from_numpy(depth_raw_np) + valid_mask_ts = torch.from_numpy(valid_mask_np) + + # 1. 保存预测深度图 + depth_pred_vis = colorize_depth_map(depth_pred_ts) + pred_save_path = os.path.join(save_vis_path, f"{safe_pred_name}{cfg_suffix}_pred.png") + depth_pred_vis.save(pred_save_path) + print(f"saved: {pred_save_path}") + # 3. 保存误差图 + # 计算绝对相对误差 + abs_rel_error = torch.abs(depth_pred_ts - depth_raw_ts) / (depth_raw_ts + 1e-6) + abs_rel_error = abs_rel_error * valid_mask_ts.float() + + # 使用matplotlib生成误差图 + import matplotlib + matplotlib.use('Agg') # 使用非交互式后端 + import matplotlib.pyplot as plt + import matplotlib.cm as cm + + error_np = abs_rel_error.numpy() + # 设置误差显示范围 + vmax = 0.2 # 可以根据需要调整 + error_normalized = np.clip(error_np / vmax, 0, 1) + + # 应用颜色映射 + jet_cmap = cm.get_cmap('jet') + error_colored = jet_cmap(error_normalized)[:, :, :3] # 去掉alpha通道 + error_colored = (error_colored * 255).astype(np.uint8) + + # 将无效区域设为黑色 + error_colored[~valid_mask_np] = [0, 0, 0] + + error_save_path = os.path.join(save_vis_path, f"{safe_pred_name}{cfg_suffix}_error.png") + plt.imsave(error_save_path, error_colored) + print(f"saved: {error_save_path}") + # 关闭matplotlib figure以释放内存 + plt.close('all') + + except Exception as e: + print(f"[VIS-Worker-{rank}] 可视化保存失败: {e}", file=sys.stderr) + + +def prepare_input_rgb_data(input_rgb): + """ + 预处理输入RGB数据，转换为numpy格式供子进程使用 + """ + if input_rgb is None: + return None + + try: + # 处理不同格式的输入图像 + if isinstance(input_rgb, torch.Tensor): + # 如果是torch tensor，转换为numpy + if input_rgb.dim() == 4: # Batch dimension + input_rgb = input_rgb[0] + if input_rgb.dim() == 3 and input_rgb.shape[0] == 3: # CHW格式 + input_rgb = input_rgb.permute(1, 2, 0) + + # 确保值在[0,1]范围内 + if input_rgb.max() <= 1.0: + input_rgb = (input_rgb * 255).clamp(0, 255).byte() + + input_rgb_np = input_rgb.cpu().numpy() + + elif isinstance(input_rgb, np.ndarray): + input_rgb_np = input_rgb + # 确保值在正确范围内 + if input_rgb_np.max() <= 1.0: + input_rgb_np = (input_rgb_np * 255).astype(np.uint8) + else: + # 假设是PIL图像或其他格式 + input_rgb_np = np.array(input_rgb) + + return input_rgb_np.copy() # 创建副本避免进程间共享问题 + + except Exception as e: + print(f"处理输入RGB数据失败: {e}", file=sys.stderr) + return None + + +def evaluate_single_prediction(pred_depth, depth_raw, valid_mask, dataset, device, metric_funcs, alignment_max_res=None, save_pred_vis=False, save_vis_path=None, pred_name=None, cfg_suffix="", alignment="least_square", rank=0, input_rgb=None): + """Args: pred_depth: 预测的深度图 (numpy array, [0,1]) depth_raw: 真实深度图 (numpy array) valid_mask: 有效掩码 (numpy array) dataset: 数据集对象 device: 计算设备 metric_funcs: 评估指标函数列表 alignment_max_res: 对齐时的最大分辨率 save_pred_vis: 是否保存可视化结果 save_vis_path: 可视化保存路径 pred_name: 预测文件名 cfg_suffix: cfg后缀，用于区分不同的cfg设置 + alignment: 对齐方式，可选值为"least_square"或"least_square_disparity" + rank: GPU rank，用于多进程图像保存 + input_rgb: 输入RGB图像 (torch.Tensor or PIL.Image or numpy.ndarray) + Returns: sample_metric: 该样本的所有评估指标列表""" + # 确保预测深度图的维度正确 + if len(pred_depth.shape) == 3: + pred_depth = pred_depth.mean(0) # [0,1] + + # 调整预测深度图尺寸以匹配真实深度图 + if pred_depth.shape != depth_raw.shape: + pred_depth = cv2.resize(pred_depth, (depth_raw.shape[1], depth_raw.shape[0]), interpolation=cv2.INTER_LINEAR) + + if "least_square" == alignment: + depth_pred, scale, shift = align_depth_least_square( + gt_arr=depth_raw, + pred_arr=pred_depth, + valid_mask_arr=valid_mask, + return_scale_shift=True, + max_resolution=alignment_max_res, + ) + elif "log_space" == alignment: + gt_log, gt_non_neg_mask = depth2log_space(depth=depth_raw, return_mask=True) + pred_non_neg_mask = pred_depth > 0 + valid_nonnegative_mask = valid_mask & gt_non_neg_mask & pred_non_neg_mask + + # 确保输入是numpy数组类型 + if isinstance(gt_log, torch.Tensor): + gt_log = gt_log.cpu().numpy() + + log_space_pred, scale, shift = align_depth_least_square( + gt_arr=gt_log, + pred_arr=pred_depth, + valid_mask_arr=valid_nonnegative_mask, + return_scale_shift=True, + max_resolution=alignment_max_res, + ) + log_space_pred = np.clip(log_space_pred, a_min=None, a_max=5.) + depth_pred = log_space2depth(log_space_pred) + # 裁剪到数据集的深度范围 + depth_pred = np.clip(depth_pred, a_min=dataset.min_depth, a_max=dataset.max_depth) + + # 裁剪到 d > 0 以便评估 + depth_pred = np.clip(depth_pred, a_min=1e-6, a_max=None) + + # 转换到设备进行评估 + depth_pred_ts = torch.from_numpy(depth_pred).to(device) + depth_raw_ts = torch.from_numpy(depth_raw).to(device) + valid_mask_ts = torch.from_numpy(valid_mask).to(device) + + # 启动可视化保存进程（同步） + if save_pred_vis and save_vis_path is not None and pred_name is not None: + safe_pred_name = pred_name.replace('/', '_').replace('\\', '_') + input_rgb_data = prepare_input_rgb_data(input_rgb) + vis_process = mp.Process( + target=save_visualization_worker, + args=( + save_vis_path, + safe_pred_name, + cfg_suffix, + depth_pred.copy(), + depth_raw.copy(), + valid_mask.copy(), + input_rgb_data, + rank + ) + ) + vis_process.start() + # save_visualization_worker(save_vis_path, safe_pred_name, cfg_suffix, depth_pred.copy(), depth_raw.copy(), valid_mask.copy(), input_rgb_data, rank) + + # 计算评估指标 + sample_metric = [] + for met_func in metric_funcs: + _metric = met_func(depth_pred_ts, depth_raw_ts, valid_mask_ts).item() + sample_metric.append(_metric) + + return sample_metric + + +def evaluation_depth_custom_parallel(rank, world_size, output_dir, dataset_config, args, pipeline, base_data_dir, pred_suffix="", alignment="least_square", alignment_max_res=None, prediction_dir=None, save_pred_vis=False): + """ + 支持多GPU并行的深度评估函数 + """ + import time + + os.makedirs(output_dir, exist_ok=True) + + cuda_avail = torch.cuda.is_available() + device = torch.device(f"cuda:{rank}") + + cfg_data = OmegaConf.load(dataset_config) + + dataset: BaseDepthDataset = get_dataset(cfg_data, base_data_dir=base_data_dir, mode=DatasetMode.EVAL, prompt_type=args.prompt_type) + + # 获取数据集名称，用于CSV表命名 + dataset_name = dataset.__class__.__name__ + + # 初始化存储结果的数据列表 + results_data = [] + + # 计算每个GPU处理的数据范围 + total_samples = len(dataset) + if args.num_samples > 0: + total_samples = min(args.num_samples, total_samples) + + chunk_size = total_samples // world_size + start_idx = rank * chunk_size + end_idx = start_idx + chunk_size if rank < world_size - 1 else total_samples + + from torch.utils.data import SubsetRandomSampler + indices = list(range(start_idx, end_idx)) + + dataloader = DataLoader(dataset, batch_size=1, num_workers=0 if args.debug else 4, pin_memory=True, sampler=SubsetRandomSampler(indices),shuffle=False) + + metric_funcs = [getattr(metric, _met) for _met in eval_metrics] + + # 为cfg=1和cfg=6分别创建metric tracker + metric_tracker_Lpred = MetricTracker(*[m.__name__ for m in metric_funcs]) + metric_tracker_Lpred.reset() + metric_tracker_Rpred = MetricTracker(*[m.__name__ for m in metric_funcs]) + metric_tracker_Rpred.reset() + + if save_pred_vis: + save_vis_path = os.path.join(output_dir, "vis") + os.makedirs(save_vis_path, exist_ok=True) + + # 创建CSV保存目录 - 每个卡都创建 + csv_save_path = os.path.join(output_dir, "csv_results") + os.makedirs(csv_save_path, exist_ok=True) + else: + save_vis_path = None + csv_save_path = None + + processing_times = [] + vis_processes = [] # 用于管理可视化进程 + max_vis_processes = 4 # 限制同时运行的可视化进程数量 + + sample_count = 0 + for data in dataloader: + sample_count += 1 + + depth_raw_ts = data["depth_raw_linear"].squeeze() + valid_mask_ts = data["valid_mask_raw"].squeeze() + rgb_name = data["rgb_relative_path"][0] + + depth_raw = depth_raw_ts.numpy() + valid_mask = valid_mask_ts.numpy() + + # Get predictions + rgb_basename = os.path.basename(rgb_name) + pred_basename = get_pred_name(rgb_basename, dataset.name_mode, suffix=pred_suffix) + pred_name = os.path.join(os.path.dirname(rgb_name), pred_basename) + + start_time = time.time() + image_list, Lpred, Rpred = pipeline.generate_image(args.prompt if args.prompt_type == "query" else data["prompt"][0], negative_prompt="", ref_images=data["rgb"], num_samples=1, num_steps=args.num_steps, cfg_guidance=args.cfg_guidance, seed=args.seed + rank, show_progress=False, size_level=args.size_level, args=args) + end_time = time.time() + processing_times.append(end_time - start_time) + + Lpred = Lpred[0].cpu().numpy() + + # 保存可视化结果（使用新的多进程方式） + if save_pred_vis and save_vis_path is not None: + # 清理文件名，替换路径分隔符 + safe_pred_name = pred_name.replace('/', '_').replace('\\', '_') + + # 预处理输入RGB数据 + input_rgb_data = prepare_input_rgb_data(data["rgb"]) + + # 限制同时运行的可视化进程数量 + while len([p for p in vis_processes if p.is_alive()]) >= max_vis_processes: + # 等待一些进程完成 + for p in vis_processes[:]: + if not p.is_alive(): + vis_processes.remove(p) + if len([p for p in vis_processes if p.is_alive()]) >= max_vis_processes: + time.sleep(0.1) # 短暂等待 + + # 创建子进程进行可视化保存 + vis_process = mp.Process( + target=save_visualization_worker, + args=( + save_vis_path, + safe_pred_name, + "_Lpred", + Lpred.copy(), + depth_raw.copy(), + valid_mask.copy(), + input_rgb_data, + rank + ) + ) + vis_process.start() + vis_processes.append(vis_process) + + sample_metric_Lpred = evaluate_single_prediction(pred_depth=Lpred, depth_raw=depth_raw, valid_mask=valid_mask, dataset=dataset, device=device, metric_funcs=metric_funcs, alignment_max_res=alignment_max_res, save_pred_vis=False, save_vis_path=None, pred_name=pred_name, cfg_suffix="_Lpred", alignment=alignment, rank=rank, input_rgb=data["rgb"]) + + for i, met_func in enumerate(metric_funcs): + metric_name = met_func.__name__ + metric_tracker_Lpred.update(metric_name, sample_metric_Lpred[i]) + + # 输出每个样本的结果 + img_id = os.path.basename(rgb_name).replace('.png', '').replace('.jpg', '') + global_sample_idx = start_idx + sample_count + + # CFG=1结果 + abs_rel_Lpred = sample_metric_Lpred[0] # abs_relative_difference + rmse_Lpred = sample_metric_Lpred[2] # rmse_linear + delta1_Lpred = sample_metric_Lpred[4] # delta1_acc + + # 修改输出格式 + if args.save_viz: + print(f"|{global_sample_idx:03d}|{abs_rel_Lpred:.4f}|{rmse_Lpred:.4f}|{delta1_Lpred:.4f}|", file=sys.stderr) + elif not args.save_viz: + print(f"[GPU:{rank}] 样本:{global_sample_idx:03d}/{total_samples} | ID:{img_id:<12}", file=sys.stderr) + print(f" CFG=1: abs_rel:{abs_rel_Lpred:.4f} | rmse:{rmse_Lpred:.4f} | a1:{delta1_Lpred:.4f}", file=sys.stderr) + print(f" 时间: {processing_times[-1]:.2f}s", file=sys.stderr) + + # 所有卡都保存结果到列表 + if args.save_viz: + results_data.append({'GPU_Rank': rank, 'Sample_ID': global_sample_idx, 'Image_Name': rgb_name, 'abs_rel': abs_rel_Lpred, 'rmse': rmse_Lpred, 'delta1': delta1_Lpred, 'processing_time': processing_times[-1]}) + + # 等待所有可视化进程完成 + if save_pred_vis: + print(f"[GPU:{rank}] 等待可视化进程完成...", file=sys.stderr) + for p in vis_processes: + p.join(timeout=30) # 设置超时时间 + if p.is_alive(): + print(f"[GPU:{rank}] 可视化进程超时，强制终止", file=sys.stderr) + p.terminate() + print(f"[GPU:{rank}] 所有可视化进程已完成", file=sys.stderr) + + if args.save_viz and csv_save_path is not None: + csv_file_path = os.path.join(csv_save_path, f"{dataset_name}_results_rank{rank}.csv") + + try: + with open(csv_file_path, 'w', newline='') as csvfile: + fieldnames = ['GPU_Rank', 'Sample_ID', 'Image_Name', 'abs_rel', 'rmse', 'delta1', 'processing_time'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for row in results_data: + writer.writerow(row) + print(f"[GPU:{rank}] 结果已保存至CSV: {csv_file_path}", file=sys.stderr) + except Exception as e: + print(f"[GPU:{rank}] 保存CSV失败: {e}", file=sys.stderr) + + return metric_tracker_Lpred, metric_tracker_Rpred, processing_times + + +def evaluation_normal_custom_parallel(rank, world_size, output_dir, base_data_dir, dataset_split_path, pipeline, args, eval_datasets, save_pred_vis=False): + """ + 支持多GPU并行的normal评估函数 + """ + import time + + os.makedirs(output_dir, exist_ok=True) + device = torch.device(f"cuda:{rank}") + + # 为每个数据集创建结果字典 + all_normal_errors = {} + all_processing_times = {} + all_dataset_metrics = {} + + for dataset_name, split in eval_datasets: + # 创建数据加载器 - 减少num_workers避免资源竞争 + try: + # 创建数据集 + from infer.dataset_normal.normal_dataloader import NormalDataset + dataset = NormalDataset(base_data_dir, dataset_split_path, dataset_name=dataset_name, split=split, mode='test', epoch=0) + + total_samples = len(dataset) + if args.num_samples > 0: + total_samples = min(args.num_samples, total_samples) + + # 计算当前GPU需要处理的样本范围 + samples_per_gpu = total_samples // world_size + start_idx = rank * samples_per_gpu + if rank == world_size - 1: + end_idx = total_samples + else: + end_idx = start_idx + samples_per_gpu + + # 创建样本索引并使用SubsetRandomSampler + from torch.utils.data import SubsetRandomSampler + indices = list(range(start_idx, end_idx)) + + dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=False, sampler=SubsetRandomSampler(indices)) + + if rank == 0: + print(f"[GPU:{rank}] 开始评估Normal数据集: {dataset_name}") + + except Exception as e: + print(f"[GPU:{rank}] 创建数据加载器失败: {e}") + continue + + dataset_output_dir = os.path.join(output_dir, dataset_name) + os.makedirs(dataset_output_dir, exist_ok=True) + + if save_pred_vis: + save_vis_path = os.path.join(dataset_output_dir, "vis") + os.makedirs(save_vis_path, exist_ok=True) + else: + save_vis_path = None + + processing_times = [] + total_normal_errors = None + sample_count = 0 + vis_processes = [] # 用于管理normal可视化进程 + max_vis_processes = 5 # 限制同时运行的可视化进程数量（normal可视化更耗内存） + + for data_dict in dataloader: + sample_count += 1 + + img = data_dict['img'].to(device) + scene_names = data_dict['scene_name'] + img_names = data_dict['img_name'] + + # 获取原始图像尺寸 + _, _, orig_H, orig_W = img.shape + + start_time = time.time() + image_list,L_pred , norm_out = pipeline.generate_image("Predict the depth map for the image on the left and the normal map on the right.", negative_prompt="", ref_images=img, num_samples=1, num_steps=args.num_steps, cfg_guidance=args.cfg_guidance, seed=args.seed + rank, show_progress=False, size_level=args.size_level, args=args, judge=data_dict['normal'].to(device) if dataset_name == "vkitti" or dataset_name == "hypersim" else None, name=img_names) + end_time = time.time() + processing_times.append(end_time - start_time) + + # 处理normal输出 + norm_out = torch.nn.functional.interpolate(norm_out, size=(orig_H, orig_W), mode='bilinear', align_corners=False) + norm = torch.linalg.norm(norm_out, axis=1, keepdims=True) + norm[norm < 1e-9] = 1e-9 + norm_out = norm_out / norm + + pred_norm, pred_kappa = norm_out[:, :3, :, :], norm_out[:, 3:, :, :] + pred_kappa = None if pred_kappa.size(1) == 0 else pred_kappa + + # 计算误差（如果有ground truth） + # if 'normal' in data_dict.keys(): + gt_norm = data_dict['normal'].to(device) + gt_norm_mask = data_dict['normal_mask'].to(device) + + pred_error = normal_utils.compute_normal_error(pred_norm, gt_norm) + if total_normal_errors is None: + total_normal_errors = pred_error[gt_norm_mask] + else: + total_normal_errors = torch.cat((total_normal_errors, pred_error[gt_norm_mask]), dim=0) + + # 保存可视化结果（使用新的多进程方式） + if save_vis_path is not None: + # 限制同时运行的可视化进程数量 + while len([p for p in vis_processes if p.is_alive()]) >= max_vis_processes: + # 等待一些进程完成 + for p in vis_processes[:]: + if not p.is_alive(): + vis_processes.remove(p) + if len([p for p in vis_processes if p.is_alive()]) >= max_vis_processes: + time.sleep(0.1) # 短暂等待 + + prefixs = ['%s_%s' % (i, j) for (i, j) in zip(scene_names, img_names)] + + # 预处理数据 + img_data, pred_norm_data, pred_kappa_data, gt_norm_data, gt_norm_mask_data, pred_error_data = prepare_normal_data_for_process( + img, pred_norm, pred_kappa, gt_norm, gt_norm_mask, pred_error + ) + + if img_data is not None: # 确保数据预处理成功 + # 创建子进程进行可视化保存 + vis_process = mp.Process( + target=save_normal_visualization_worker, + args=( + save_vis_path, + prefixs, + img_data, + pred_norm_data, + pred_kappa_data, + gt_norm_data, + gt_norm_mask_data, + pred_error_data, + rank + ) + ) + vis_process.start() + vis_processes.append(vis_process) + + # 输出进度信息 + global_sample_idx = start_idx + sample_count + img_id = '_'.join([scene_names[0], img_names[0]]) + + if rank == 0 or sample_count % 10 == 0: # 减少输出频率 + print(f"[GPU:{rank}] | 样本:{global_sample_idx:03d} | ID:{img_id} | 时间:{processing_times[-1]:.2f}s| ", file=sys.stderr) + + # 等待所有可视化进程结束 + if save_pred_vis: + print(f"[GPU:{rank}] 等待Normal可视化进程完成...", file=sys.stderr) + for p in vis_processes: + p.join(timeout=60) # normal可视化需要更长时间，设置60秒超时 + if p.is_alive(): + print(f"[GPU:{rank}] Normal可视化进程超时，强制终止", file=sys.stderr) + p.terminate() + print(f"[GPU:{rank}] 所有Normal可视化进程已完成", file=sys.stderr) + + # 计算当前GPU的指标 + metrics = None + if total_normal_errors is not None and len(total_normal_errors) > 0: + metrics = normal_utils.compute_normal_metrics(total_normal_errors) + if rank == 0: + print(f"[GPU:{rank}] 数据集 {dataset_name} 部分结果:") + print("mean median rmse 5 7.5 11.25 22.5 30") + print("%.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f" % (metrics['mean'], metrics['median'], metrics['rmse'], metrics['a1'], metrics['a2'], metrics['a3'], metrics['a4'], metrics['a5'])) + + # 存储结果 + all_normal_errors[dataset_name] = total_normal_errors.cpu() if total_normal_errors is not None else None + all_processing_times[dataset_name] = processing_times + all_dataset_metrics[dataset_name] = metrics + + return all_normal_errors, all_processing_times, all_dataset_metrics + +def save_normal_visualization_worker(save_vis_path, prefixs, img_data, pred_norm_data, pred_kappa_data, gt_norm_data, gt_norm_mask_data, pred_error_data, rank): + """ + Normal可视化保存的工作函数，在独立进程中运行 + Args: + save_vis_path: 保存路径 + prefixs: 文件名前缀列表 + img_data: 输入图像数据 (numpy array) + pred_norm_data: 预测normal数据 (numpy array) + pred_kappa_data: 预测kappa数据 (numpy array or None) + gt_norm_data: GT normal数据 (numpy array) + gt_norm_mask_data: GT normal掩码数据 (numpy array) + pred_error_data: 预测误差数据 (numpy array) + rank: GPU rank + """ + try: + import infer.visualize as vis_utils + + # 转换为torch tensor用于可视化函数 + img_ts = torch.from_numpy(img_data) + pred_norm_ts = torch.from_numpy(pred_norm_data) + pred_kappa_ts = torch.from_numpy(pred_kappa_data) if pred_kappa_data is not None else None + gt_norm_ts = torch.from_numpy(gt_norm_data) + gt_norm_mask_ts = torch.from_numpy(gt_norm_mask_data) + pred_error_ts = torch.from_numpy(pred_error_data) + + # 使用matplotlib的非交互式后端 + import matplotlib + matplotlib.use('Agg') + + # 调用可视化函数 + vis_utils.visualize_normal(save_vis_path, prefixs, img_ts, pred_norm_ts, pred_kappa_ts, gt_norm_ts, gt_norm_mask_ts, pred_error_ts) + + except Exception as e: + print(f"[NORMAL-VIS-Worker-{rank}] Normal可视化保存失败: {e}", file=sys.stderr) + + +def prepare_normal_data_for_process(img, pred_norm, pred_kappa, gt_norm, gt_norm_mask, pred_error): + """ + 预处理normal数据，转换为numpy格式供子进程使用 + """ + try: + img_data = img.cpu().numpy() + pred_norm_data = pred_norm.cpu().numpy() + pred_kappa_data = pred_kappa.cpu().numpy() if pred_kappa is not None else None + gt_norm_data = gt_norm.cpu().numpy() + gt_norm_mask_data = gt_norm_mask.cpu().numpy() + pred_error_data = pred_error.cpu().numpy() + + return img_data, pred_norm_data, pred_kappa_data, gt_norm_data, gt_norm_mask_data, pred_error_data + except Exception as e: + print(f"处理Normal数据失败: {e}", file=sys.stderr) + return None, None, None, None, None, None diff --git a/infer/sampling.py b/infer/sampling.py new file mode 100644 index 0000000000000000000000000000000000000000..16142b95c9165d3e645d5eabb2f2524481df068e --- /dev/null +++ b/infer/sampling.py @@ -0,0 +1,47 @@ +import math +from collections.abc import Callable + +import torch +from torch import Tensor + + +def get_noise(num_samples: int, height: int, width: int, device: torch.device, dtype: torch.dtype, seed: int): + return torch.randn( + num_samples, + 16, + # allow for packing + 2 * math.ceil(height / 16), + 2 * math.ceil(width / 16), + device=device, + dtype=dtype, + generator=torch.Generator(device=device).manual_seed(seed), + ) + + +def time_shift(mu: float, sigma: float, t: Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + +def get_lin_function(x1: float = 256, y1: float = 0.5, x2: float = 4096, y2: float = 1.15) -> Callable[[float], float]: + m = (y2 - y1) / (x2 - x1) + b = y1 - m * x1 + return lambda x: m * x + b + + +def get_schedule( + num_steps: int, + image_seq_len: int, + base_shift: float = 0.5, + max_shift: float = 1.15, + shift: bool = True, +) -> list[float]: + # extra step for zero + timesteps = torch.linspace(1, 0, num_steps + 1) + + # shifting the schedule to favor high timesteps for higher signal images + if shift: + # estimate mu based on linear estimation between two points + mu = get_lin_function(y1=base_shift, y2=max_shift)(image_seq_len) + timesteps = time_shift(mu, 1.0, timesteps) + + return timesteps.tolist() diff --git a/infer/seed_all.py b/infer/seed_all.py new file mode 100644 index 0000000000000000000000000000000000000000..95795654e36d3fbc14e876dff50f348a1f5a9c0a --- /dev/null +++ b/infer/seed_all.py @@ -0,0 +1,33 @@ +# Copyright 2023 Bingxin Ke, ETH Zurich. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------------------------- +# If you find this code useful, we kindly ask you to cite our paper in your work. +# Please find bibtex at: https://github.com/prs-eth/Marigold#-citation +# More information about the method can be found at https://marigoldmonodepth.github.io +# -------------------------------------------------------------------------- + + +import numpy as np +import random +import torch + + +def seed_all(seed: int = 0): + """ + Set random seeds of all components. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) diff --git a/infer/util/__init__.py b/infer/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infer/util/alignment.py b/infer/util/alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..21720fc389f1756da844045b42bb71ce812c45a0 --- /dev/null +++ b/infer/util/alignment.py @@ -0,0 +1,88 @@ +# Author: Bingxin Ke +# Last modified: 2024-01-11 + +import numpy as np +import torch + + +def align_depth_least_square( + gt_arr: np.ndarray, + pred_arr: np.ndarray, + valid_mask_arr: np.ndarray, + return_scale_shift=True, + max_resolution=None, +): + ori_shape = pred_arr.shape # input shape + + gt = gt_arr.squeeze() # [H, W] + pred = pred_arr.squeeze() + valid_mask = valid_mask_arr.squeeze() + + # Downsample + if max_resolution is not None: + scale_factor = np.min(max_resolution / np.array(ori_shape[-2:])) + if scale_factor < 1: + downscaler = torch.nn.Upsample(scale_factor=scale_factor, mode="nearest") + gt = downscaler(torch.as_tensor(gt).unsqueeze(0)).numpy() + pred = downscaler(torch.as_tensor(pred).unsqueeze(0)).numpy() + valid_mask = ( + downscaler(torch.as_tensor(valid_mask).unsqueeze(0).float()) + .bool() + .numpy() + ) + + assert ( + gt.shape == pred.shape == valid_mask.shape + ), f"{gt.shape}, {pred.shape}, {valid_mask.shape}" + + gt_masked = gt[valid_mask].reshape((-1, 1)) + pred_masked = pred[valid_mask].reshape((-1, 1)) + + # numpy solver + _ones = np.ones_like(pred_masked) + A = np.concatenate([pred_masked, _ones], axis=-1) + X = np.linalg.lstsq(A, gt_masked, rcond=None)[0] + scale, shift = X + + aligned_pred = pred_arr * scale + shift + + # restore dimensions + aligned_pred = aligned_pred.reshape(ori_shape) + + if return_scale_shift: + return aligned_pred, scale, shift + else: + return aligned_pred + + +# ******************** disparity space ******************** +def depth2disparity(depth, return_mask=False): + if isinstance(depth, torch.Tensor): + disparity = torch.zeros_like(depth) + elif isinstance(depth, np.ndarray): + disparity = np.zeros_like(depth) + non_negtive_mask = depth > 0 + disparity[non_negtive_mask] = 1.0 / depth[non_negtive_mask] + if return_mask: + return disparity, non_negtive_mask + else: + return disparity + + +def disparity2depth(disparity, **kwargs): + return depth2disparity(disparity, **kwargs) + +# ******************** log space ******************** +def depth2log_space(depth, **kwargs): + if isinstance(depth, torch.Tensor): + log_space = torch.zeros_like(depth) + elif isinstance(depth, np.ndarray): + log_space = np.zeros_like(depth) + non_negtive_mask = depth > 0 + log_space[non_negtive_mask] = np.log(depth[non_negtive_mask]) + return log_space, non_negtive_mask + +def log_space2depth(log_space, **kwargs): + depth = np.exp(log_space) + return depth + diff --git a/infer/util/metric.py b/infer/util/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..b6f16b8c0379f252f98dbc80f33aca086796fa73 --- /dev/null +++ b/infer/util/metric.py @@ -0,0 +1,151 @@ +import pandas as pd +import torch + + +# Adapted from: https://github.com/victoresque/pytorch-template/blob/master/utils/util.py +class MetricTracker: + def __init__(self, *keys, writer=None): + self.writer = writer + self._data = pd.DataFrame(index=keys, columns=["total", "counts", "average"]) + self.reset() + + def reset(self): + for col in self._data.columns: + self._data[col].values[:] = 0.0 + + def update(self, key, value, n=1): + if self.writer is not None: + self.writer.add_scalar(key, value) + # 确保value是数值类型 + value = float(value) if hasattr(value, '__float__') else value.item() if hasattr(value, 'item') else float(value) + self._data.at[key, "total"] += value * n + self._data.at[key, "counts"] += n + self._data.at[key, "average"] = self._data.at[key, "total"] / self._data.at[key, "counts"] + + def avg(self, key): + return self._data.average[key] + + def result(self): + return dict(self._data.average) + +def pixel_mean(pred, gt, valid_mask): + if valid_mask is not None: + masked_pred = pred * valid_mask + masked_gt = gt * valid_mask + + valid_pixel_count = torch.sum(valid_mask, dim=(0,1)) + + pred_mean = torch.sum(masked_pred, dim=(0,1)) / valid_pixel_count + gt_mean = torch.sum(masked_gt, dim=(0,1)) / valid_pixel_count + else: + pred_mean = torch.mean(pred, dim=(0,1)) + gt_mean = torch.mean(gt, dim=(0,1)) + + mean_difference = torch.abs(pred_mean - gt_mean) + return mean_difference + +def pixel_var(pred, gt, valid_mask): + if valid_mask is not None: + masked_pred = pred * valid_mask + masked_gt = gt * valid_mask + + valid_pixel_count = torch.sum(valid_mask, dim=(0,1)) + + pred_mean = torch.sum(masked_pred, dim=(0,1)) / valid_pixel_count + gt_mean = torch.sum(masked_gt, dim=(0,1)) / valid_pixel_count + + pred_var = torch.sum(valid_mask * (pred - pred_mean)**2, dim=(0,1)) / valid_pixel_count + gt_var = torch.sum(valid_mask * (gt - gt_mean)**2, dim=(0,1)) / valid_pixel_count + else: + pred_var = torch.var(pred, dim=(0,1)) + gt_var = torch.var(gt, dim=(0,1)) + + var_difference = torch.abs(pred_var - gt_var) + + return var_difference + +def abs_relative_difference(output, target, valid_mask=None): + actual_output = output + actual_target = target + abs_relative_diff = torch.abs(actual_output - actual_target) / actual_target + if valid_mask is not None: + abs_relative_diff[~valid_mask] = 0 + n = valid_mask.sum((-1, -2)) + else: + n = output.shape[-1] * output.shape[-2] + abs_relative_diff = torch.sum(abs_relative_diff, (-1, -2)) / n + return abs_relative_diff.mean() + + +def squared_relative_difference(output, target, valid_mask=None): + actual_output = output + actual_target = target + square_relative_diff = ( + torch.pow(torch.abs(actual_output - actual_target), 2) / actual_target + ) + if valid_mask is not None: + square_relative_diff[~valid_mask] = 0 + n = valid_mask.sum((-1, -2)) + else: + n = output.shape[-1] * output.shape[-2] + square_relative_diff = torch.sum(square_relative_diff, (-1, -2)) / n + return square_relative_diff.mean() + + +def rmse_linear(output, target, valid_mask=None): + actual_output = output + actual_target = target + diff = actual_output - actual_target + if valid_mask is not None: + diff[~valid_mask] = 0 + n = valid_mask.sum((-1, -2)) + else: + n = output.shape[-1] * output.shape[-2] + diff2 = torch.pow(diff, 2) + mse = torch.sum(diff2, (-1, -2)) / n + rmse = torch.sqrt(mse) + return rmse.mean() + + +def rmse_log(output, target, valid_mask=None): + diff = torch.log(output) - torch.log(target) + if valid_mask is not None: + diff[~valid_mask] = 0 + n = valid_mask.sum((-1, -2)) + else: + n = output.shape[-1] * output.shape[-2] + diff2 = torch.pow(diff, 2) + mse = torch.sum(diff2, (-1, -2)) / n # [B] + rmse = torch.sqrt(mse) + return rmse.mean() + + + + +# adapt from: https://github.com/imran3180/depth-map-prediction/blob/master/main.py +def threshold_percentage(output, target, threshold_val, valid_mask=None): + d1 = output / target + d2 = target / output + max_d1_d2 = torch.max(d1, d2) + bit_mat = (max_d1_d2 < threshold_val).to(output.dtype) + if valid_mask is not None: + bit_mat = bit_mat * valid_mask.to(output.dtype) + n = valid_mask.sum((-1, -2)) + else: + n = torch.tensor(output.shape[-1] * output.shape[-2], device=output.device) + n = torch.clamp(n, min=1) + count_mat = torch.sum(bit_mat, (-1, -2)) + threshold_mat = count_mat / n.to(count_mat.dtype) + return threshold_mat.mean() + + +def delta1_acc(pred, gt, valid_mask): + return threshold_percentage(pred, gt, 1.25, valid_mask) + + +def delta2_acc(pred, gt, valid_mask): + return threshold_percentage(pred, gt, 1.25**2, valid_mask) + + +def delta3_acc(pred, gt, valid_mask): + return threshold_percentage(pred, gt, 1.25**3, valid_mask) diff --git a/infer/util/normal_utils.py b/infer/util/normal_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..14b192b4d3c970c76070a2d18e00da841a228dab --- /dev/null +++ b/infer/util/normal_utils.py @@ -0,0 +1,78 @@ +import os +import numpy as np +import torch +import torch.nn.functional as F +import torch.distributed as dist + + +def get_padding(orig_H, orig_W): + """ returns how the input of shape (orig_H, orig_W) should be padded + this ensures that both H and W are divisible by 32 + """ + if orig_W % 32 == 0: + l = 0 + r = 0 + else: + new_W = 32 * ((orig_W // 32) + 1) + l = (new_W - orig_W) // 2 + r = (new_W - orig_W) - l + + if orig_H % 32 == 0: + t = 0 + b = 0 + else: + new_H = 32 * ((orig_H // 32) + 1) + t = (new_H - orig_H) // 2 + b = (new_H - orig_H) - t + return l, r, t, b + +def pad_input(img, intrins, lrtb=(0,0,0,0)): + """ pad input image + img should be a torch tensor of shape (B, 3, H, W) + intrins should be a torch tensor of shape (B, 3, 3) + """ + l, r, t, b = lrtb + if l+r+t+b != 0: + pad_value_R = (0 - 0.485) / 0.229 + pad_value_G = (0 - 0.456) / 0.224 + pad_value_B = (0 - 0.406) / 0.225 + + img_R = F.pad(img[:,0:1,:,:], (l, r, t, b), mode="constant", value=pad_value_R) + img_G = F.pad(img[:,1:2,:,:], (l, r, t, b), mode="constant", value=pad_value_G) + img_B = F.pad(img[:,2:3,:,:], (l, r, t, b), mode="constant", value=pad_value_B) + + img = torch.cat([img_R, img_G, img_B], dim=1) + + if intrins is not None: + intrins[:, 0, 2] += l + intrins[:, 1, 2] += t + return img, intrins + +def compute_normal_error(pred_norm, gt_norm): + """ compute per-pixel surface normal error in degrees + NOTE: pred_norm and gt_norm should be torch tensors of shape (B, 3, ...) + """ + pred_error = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + pred_error = torch.clamp(pred_error, min=-1.0, max=1.0) + pred_error = torch.acos(pred_error) * 180.0 / np.pi + pred_error = pred_error.unsqueeze(1) # (B, 1, ...) + return pred_error + +def compute_normal_metrics(total_normal_errors): + """ compute surface normal metrics (used for benchmarking) + NOTE: total_normal_errors should be a 1D torch tensor of errors in degrees + """ + total_normal_errors = total_normal_errors.detach().cpu().numpy() + num_pixels = total_normal_errors.shape[0] + + metrics = { + 'mean': np.average(total_normal_errors), + 'median': np.median(total_normal_errors), + 'rmse': np.sqrt(np.sum(total_normal_errors * total_normal_errors) / num_pixels), + 'a1': 100.0 * (np.sum(total_normal_errors < 5) / num_pixels), + 'a2': 100.0 * (np.sum(total_normal_errors < 7.5) / num_pixels), + 'a3': 100.0 * (np.sum(total_normal_errors < 11.25) / num_pixels), + 'a4': 100.0 * (np.sum(total_normal_errors < 22.5) / num_pixels), + 'a5': 100.0 * (np.sum(total_normal_errors < 30) / num_pixels) + } + return metrics \ No newline at end of file diff --git a/infer/visualize.py b/infer/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..c99e18b1c0835a6e6b3f5b935a736f9fed7a135a --- /dev/null +++ b/infer/visualize.py @@ -0,0 +1,130 @@ +import cv2 +import numpy as np + +import torch + +from matplotlib import cm +import matplotlib.pyplot as plt + +import logging +logger = logging.getLogger('root') + + + +def tensor_to_numpy(tensor_in): + """ torch tensor to numpy array + """ + if tensor_in is not None: + if tensor_in.ndim == 3: + # (C, H, W) -> (H, W, C) + tensor_in = tensor_in.detach().cpu().permute(1, 2, 0).numpy() + elif tensor_in.ndim == 4: + # (B, C, H, W) -> (B, H, W, C) + tensor_in = tensor_in.detach().cpu().permute(0, 2, 3, 1).numpy() + else: + raise Exception('invalid tensor size') + return tensor_in + +# def unnormalize(img_in, img_stats={'mean': [0.485, 0.456, 0.406], +# 'std': [0.229, 0.224, 0.225]}): +def unnormalize(img_in, img_stats={'mean': [0.5,0.5,0.5], 'std': [0.5,0.5,0.5]}): + """ unnormalize input image + """ + if torch.is_tensor(img_in): + img_in = tensor_to_numpy(img_in) + + # 检查输入图像的数值范围，决定是否需要去归一化 + img_min, img_max = img_in.min(), img_in.max() + + # 如果图像已经在[0,1]范围内，直接转换为[0,255] + if img_min >= -0.1 and img_max <= 1.1: # 允许小的浮点误差 + img_out = np.clip(img_in, 0, 1) + img_out = (img_out * 255.0).astype(np.uint8) + else: + # 如果图像在[-1,1]或其他归一化范围内，进行标准去归一化 + img_out = np.zeros_like(img_in) + for ich in range(3): + img_out[..., ich] = img_in[..., ich] * img_stats['std'][ich] + img_out[..., ich] += img_stats['mean'][ich] + img_out = np.clip(img_out, 0, 1) + img_out = (img_out * 255.0).astype(np.uint8) + + return img_out + +def normal_to_rgb(normal, normal_mask=None): + """ surface normal map to RGB + (used for visualization) + + NOTE: x, y, z are mapped to R, G, B + NOTE: [-1, 1] are mapped to [0, 255] + """ + if torch.is_tensor(normal): + normal = tensor_to_numpy(normal) + normal_mask = tensor_to_numpy(normal_mask) + + normal_norm = np.linalg.norm(normal, axis=-1, keepdims=True) + normal_norm[normal_norm < 1e-12] = 1e-12 + normal = normal / normal_norm + + normal_rgb = (((normal + 1) * 0.5) * 255).astype(np.uint8) + if normal_mask is not None: + normal_rgb = normal_rgb * normal_mask # (B, H, W, 3) + return normal_rgb + +def kappa_to_alpha(pred_kappa, to_numpy=True): + """ Confidence kappa to uncertainty alpha + Assuming AngMF distribution (introduced in https://arxiv.org/abs/2109.09881) + """ + if torch.is_tensor(pred_kappa) and to_numpy: + pred_kappa = tensor_to_numpy(pred_kappa) + + if torch.is_tensor(pred_kappa): + alpha = ((2 * pred_kappa) / ((pred_kappa ** 2.0) + 1)) \ + + ((torch.exp(- pred_kappa * np.pi) * np.pi) / (1 + torch.exp(- pred_kappa * np.pi))) + alpha = torch.rad2deg(alpha) + else: + alpha = ((2 * pred_kappa) / ((pred_kappa ** 2.0) + 1)) \ + + ((np.exp(- pred_kappa * np.pi) * np.pi) / (1 + np.exp(- pred_kappa * np.pi))) + alpha = np.degrees(alpha) + + return alpha + + +def visualize_normal(target_dir, prefixs, img, pred_norm, pred_kappa, + gt_norm, gt_norm_mask, pred_error, num_vis=-1): + """ visualize normal + """ + error_max = 60.0 + + # img = tensor_to_numpy(img) # (B, H, W, 3) + pred_norm = tensor_to_numpy(pred_norm) # (B, H, W, 3) + # pred_kappa = tensor_to_numpy(pred_kappa) # (B, H, W, 1) + gt_norm = tensor_to_numpy(gt_norm) # (B, H, W, 3) + gt_norm_mask = tensor_to_numpy(gt_norm_mask) # (B, H, W, 1) + pred_error = tensor_to_numpy(pred_error) # (B, H, W, 1) + + num_vis = len(prefixs) if num_vis == -1 else num_vis + for i in range(num_vis): + # # img + # img_ = unnormalize(img[i, ...]) + # target_path = '%s/%s_img.png' % (target_dir, prefixs[i]) + # plt.imsave(target_path, img_) + + # pred_norm + target_path = '%s/%s_norm.png' % (target_dir, prefixs[i]) + plt.imsave(target_path, normal_to_rgb(pred_norm[i, ...])) + + # # pred_kappa + # if pred_kappa is not None: + # pred_alpha = kappa_to_alpha(pred_kappa[i, :, :, 0]) + # target_path = '%s/%s_pred_alpha.png' % (target_dir, prefixs[i]) + # plt.imsave(target_path, pred_alpha, vmin=0.0, vmax=error_max, cmap='jet') + + # gt_norm, pred_error + if gt_norm is not None: + target_path = '%s/%s_gt.png' % (target_dir, prefixs[i]) + # plt.imsave(target_path, normal_to_rgb(gt_norm[i, ...], gt_norm_mask[i, ...])) + + E = pred_error[i, :, :, 0] * gt_norm_mask[i, :, :, 0] + target_path = '%s/%s_pred_error.png' % (target_dir, prefixs[i]) + plt.imsave(target_path, E, vmin=0, vmax=error_max, cmap='jet') diff --git a/latent/no_info.npz b/latent/no_info.npz new file mode 100644 index 0000000000000000000000000000000000000000..59734b99ba2f2ea9dcc774f7f82f28fe8b154def --- /dev/null +++ b/latent/no_info.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03d72f486f1cefc0bcd6d21d420156cb3e96ec9d701af667386bbd2eedbafa2 +size 9180668 diff --git a/library/__init__.py b/library/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/library/custom_offloading_utils.py b/library/custom_offloading_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..84c2b743ec55caa93b4ead696b09a5cae94f913d --- /dev/null +++ b/library/custom_offloading_utils.py @@ -0,0 +1,227 @@ +from concurrent.futures import ThreadPoolExecutor +import time +from typing import Optional +import torch +import torch.nn as nn + +from library.device_utils import clean_memory_on_device + + +def synchronize_device(device: torch.device): + if device.type == "cuda": + torch.cuda.synchronize() + elif device.type == "xpu": + torch.xpu.synchronize() + elif device.type == "mps": + torch.mps.synchronize() + + +def swap_weight_devices_cuda(device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module): + assert layer_to_cpu.__class__ == layer_to_cuda.__class__ + + weight_swap_jobs = [] + + # This is not working for all cases (e.g. SD3), so we need to find the corresponding modules + # for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()): + # print(module_to_cpu.__class__, module_to_cuda.__class__) + # if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None: + # weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)) + + modules_to_cpu = {k: v for k, v in layer_to_cpu.named_modules()} + for module_to_cuda_name, module_to_cuda in layer_to_cuda.named_modules(): + if hasattr(module_to_cuda, "weight") and module_to_cuda.weight is not None: + module_to_cpu = modules_to_cpu.get(module_to_cuda_name, None) + if module_to_cpu is not None and module_to_cpu.weight.shape == module_to_cuda.weight.shape: + weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)) + else: + if module_to_cuda.weight.data.device.type != device.type: + # print( + # f"Module {module_to_cuda_name} not found in CPU model or shape mismatch, so not swapping and moving to device" + # ) + module_to_cuda.weight.data = module_to_cuda.weight.data.to(device) + + torch.cuda.current_stream().synchronize() # this prevents the illegal loss value + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + # cuda to cpu + for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: + cuda_data_view.record_stream(stream) + module_to_cpu.weight.data = cuda_data_view.data.to("cpu", non_blocking=True) + + stream.synchronize() + + # cpu to cuda + for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: + cuda_data_view.copy_(module_to_cuda.weight.data, non_blocking=True) + module_to_cuda.weight.data = cuda_data_view + + stream.synchronize() + torch.cuda.current_stream().synchronize() # this prevents the illegal loss value + + +def swap_weight_devices_no_cuda(device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module): + """ + not tested + """ + assert layer_to_cpu.__class__ == layer_to_cuda.__class__ + + weight_swap_jobs = [] + for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()): + if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None: + weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)) + + # device to cpu + for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: + module_to_cpu.weight.data = cuda_data_view.data.to("cpu", non_blocking=True) + + synchronize_device() + + # cpu to device + for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: + cuda_data_view.copy_(module_to_cuda.weight.data, non_blocking=True) + module_to_cuda.weight.data = cuda_data_view + + synchronize_device() + + +def weighs_to_device(layer: nn.Module, device: torch.device): + for module in layer.modules(): + if hasattr(module, "weight") and module.weight is not None: + module.weight.data = module.weight.data.to(device, non_blocking=True) + + +class Offloader: + """ + common offloading class + """ + + def __init__(self, num_blocks: int, blocks_to_swap: int, device: torch.device, debug: bool = False): + self.num_blocks = num_blocks + self.blocks_to_swap = blocks_to_swap + self.device = device + self.debug = debug + + self.thread_pool = ThreadPoolExecutor(max_workers=1) + self.futures = {} + self.cuda_available = device.type == "cuda" + + def swap_weight_devices(self, block_to_cpu: nn.Module, block_to_cuda: nn.Module): + if self.cuda_available: + swap_weight_devices_cuda(self.device, block_to_cpu, block_to_cuda) + else: + swap_weight_devices_no_cuda(self.device, block_to_cpu, block_to_cuda) + + def _submit_move_blocks(self, blocks, block_idx_to_cpu, block_idx_to_cuda): + def move_blocks(bidx_to_cpu, block_to_cpu, bidx_to_cuda, block_to_cuda): + if self.debug: + start_time = time.perf_counter() + print(f"Move block {bidx_to_cpu} to CPU and block {bidx_to_cuda} to {'CUDA' if self.cuda_available else 'device'}") + + self.swap_weight_devices(block_to_cpu, block_to_cuda) + + if self.debug: + print(f"Moved blocks {bidx_to_cpu} and {bidx_to_cuda} in {time.perf_counter()-start_time:.2f}s") + return bidx_to_cpu, bidx_to_cuda # , event + + block_to_cpu = blocks[block_idx_to_cpu] + block_to_cuda = blocks[block_idx_to_cuda] + + self.futures[block_idx_to_cuda] = self.thread_pool.submit( + move_blocks, block_idx_to_cpu, block_to_cpu, block_idx_to_cuda, block_to_cuda + ) + + def _wait_blocks_move(self, block_idx): + if block_idx not in self.futures: + return + + if self.debug: + print(f"Wait for block {block_idx}") + start_time = time.perf_counter() + + future = self.futures.pop(block_idx) + _, bidx_to_cuda = future.result() + + assert block_idx == bidx_to_cuda, f"Block index mismatch: {block_idx} != {bidx_to_cuda}" + + if self.debug: + print(f"Waited for block {block_idx}: {time.perf_counter()-start_time:.2f}s") + + +class ModelOffloader(Offloader): + """ + supports forward offloading + """ + + def __init__(self, blocks: list[nn.Module], num_blocks: int, blocks_to_swap: int, device: torch.device, debug: bool = False): + super().__init__(num_blocks, blocks_to_swap, device, debug) + + # register backward hooks + self.remove_handles = [] + for i, block in enumerate(blocks): + hook = self.create_backward_hook(blocks, i) + if hook is not None: + handle = block.register_full_backward_hook(hook) + self.remove_handles.append(handle) + + def __del__(self): + for handle in self.remove_handles: + handle.remove() + + def create_backward_hook(self, blocks: list[nn.Module], block_index: int) -> Optional[callable]: + # -1 for 0-based index + num_blocks_propagated = self.num_blocks - block_index - 1 + swapping = num_blocks_propagated > 0 and num_blocks_propagated <= self.blocks_to_swap + waiting = block_index > 0 and block_index <= self.blocks_to_swap + + if not swapping and not waiting: + return None + + # create hook + block_idx_to_cpu = self.num_blocks - num_blocks_propagated + block_idx_to_cuda = self.blocks_to_swap - num_blocks_propagated + block_idx_to_wait = block_index - 1 + + def backward_hook(module, grad_input, grad_output): + if self.debug: + print(f"Backward hook for block {block_index}") + + if swapping: + self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda) + if waiting: + self._wait_blocks_move(block_idx_to_wait) + return None + + return backward_hook + + def prepare_block_devices_before_forward(self, blocks: list[nn.Module]): + if self.blocks_to_swap is None or self.blocks_to_swap == 0: + return + + if self.debug: + print("Prepare block devices before forward") + + for b in blocks[0 : self.num_blocks - self.blocks_to_swap]: + b.to(self.device) + weighs_to_device(b, self.device) # make sure weights are on device + + for b in blocks[self.num_blocks - self.blocks_to_swap :]: + b.to(self.device) # move block to device first + weighs_to_device(b, "cpu") # make sure weights are on cpu + + synchronize_device(self.device) + clean_memory_on_device(self.device) + + def wait_for_block(self, block_idx: int): + if self.blocks_to_swap is None or self.blocks_to_swap == 0: + return + self._wait_blocks_move(block_idx) + + def submit_move_blocks(self, blocks: list[nn.Module], block_idx: int): + if self.blocks_to_swap is None or self.blocks_to_swap == 0: + return + if block_idx >= self.blocks_to_swap: + return + block_idx_to_cpu = block_idx + block_idx_to_cuda = self.num_blocks - self.blocks_to_swap + block_idx + self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda) diff --git a/library/device_utils.py b/library/device_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d2e1974504fcc6010100b1c55222144fb596a54b --- /dev/null +++ b/library/device_utils.py @@ -0,0 +1,89 @@ +import functools +import gc + +import torch +try: + # intel gpu support for pytorch older than 2.5 + # ipex is not needed after pytorch 2.5 + import intel_extension_for_pytorch as ipex # noqa +except Exception: + pass + + +try: + HAS_CUDA = torch.cuda.is_available() +except Exception: + HAS_CUDA = False + +try: + HAS_MPS = torch.backends.mps.is_available() +except Exception: + HAS_MPS = False + +try: + HAS_XPU = torch.xpu.is_available() +except Exception: + HAS_XPU = False + + +def clean_memory(): + gc.collect() + if HAS_CUDA: + torch.cuda.empty_cache() + if HAS_XPU: + torch.xpu.empty_cache() + if HAS_MPS: + torch.mps.empty_cache() + + +def clean_memory_on_device(device: torch.device): + r""" + Clean memory on the specified device, will be called from training scripts. + """ + gc.collect() + + # device may "cuda" or "cuda:0", so we need to check the type of device + if device.type == "cuda": + torch.cuda.empty_cache() + if device.type == "xpu": + torch.xpu.empty_cache() + if device.type == "mps": + torch.mps.empty_cache() + + +@functools.lru_cache(maxsize=None) +def get_preferred_device() -> torch.device: + r""" + Do not call this function from training scripts. Use accelerator.device instead. + """ + if HAS_CUDA: + device = torch.device("cuda") + elif HAS_XPU: + device = torch.device("xpu") + elif HAS_MPS: + device = torch.device("mps") + else: + device = torch.device("cpu") + print(f"get_preferred_device() -> {device}") + return device + + +def init_ipex(): + """ + Apply IPEX to CUDA hijacks using `library.ipex.ipex_init`. + + This function should run right after importing torch and before doing anything else. + + If xpu is not available, this function does nothing. + """ + try: + if HAS_XPU: + from library.ipex import ipex_init + + is_initialized, error_message = ipex_init() + if not is_initialized: + print("failed to initialize ipex:", error_message) + else: + return + except Exception as e: + print("failed to initialize ipex:", e) diff --git a/library/lora_module.py b/library/lora_module.py new file mode 100644 index 0000000000000000000000000000000000000000..c5e3052eb3fa5313f09282aa5f81a8ef55bf7a62 --- /dev/null +++ b/library/lora_module.py @@ -0,0 +1,1289 @@ +# temporary minimum implementation of LoRA +# TODO commonize with the original implementation + +# LoRA network module +# reference: +# https://github.com/microsoft/LoRA/blob/main/loralib/layers.py +# https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py + +import math +import os +from contextlib import contextmanager +from typing import Dict, List, Optional, Tuple, Type, Union +from diffusers import AutoencoderKL +import numpy as np +import torch +from torch import Tensor +import re +from library.utils import setup_logging + +setup_logging() +import logging + +logger = logging.getLogger(__name__) + + +NUM_DOUBLE_BLOCKS = 19 +NUM_SINGLE_BLOCKS = 38 + + +class LoRAModule(torch.nn.Module): + """ + replaces forward method of the original Linear, instead of replacing the original Linear module. + """ + + def __init__( + self, + lora_name, + org_module: torch.nn.Module, + multiplier=1.0, + lora_dim=4, + alpha=1, + dropout=None, + rank_dropout=None, + module_dropout=None, + split_dims: Optional[List[int]] = None, + ggpo_beta: Optional[float] = None, + ggpo_sigma: Optional[float] = None, + ): + """ + if alpha == 0 or None, alpha is rank (no scaling). + """ + super().__init__() + self.lora_name = lora_name + + if org_module.__class__.__name__ == "Conv2d": + in_dim = org_module.in_channels + out_dim = org_module.out_channels + else: + in_dim = org_module.in_features + out_dim = org_module.out_features + + self.lora_dim = lora_dim + self.split_dims = split_dims + + if split_dims is None: + if org_module.__class__.__name__ == "Conv2d": + kernel_size = org_module.kernel_size + stride = org_module.stride + padding = org_module.padding + self.lora_down = torch.nn.Conv2d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False) + self.lora_up = torch.nn.Conv2d(self.lora_dim, out_dim, (1, 1), (1, 1), bias=False) + else: + self.lora_down = torch.nn.Linear(in_dim, self.lora_dim, bias=False) + self.lora_up = torch.nn.Linear(self.lora_dim, out_dim, bias=False) + + torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5)) + torch.nn.init.zeros_(self.lora_up.weight) + else: + # conv2d not supported + assert sum(split_dims) == out_dim, "sum of split_dims must be equal to out_dim" + assert org_module.__class__.__name__ == "Linear", "split_dims is only supported for Linear" + # print(f"split_dims: {split_dims}") + self.lora_down = torch.nn.ModuleList( + [torch.nn.Linear(in_dim, self.lora_dim, bias=False) for _ in range(len(split_dims))] + ) + self.lora_up = torch.nn.ModuleList([torch.nn.Linear(self.lora_dim, split_dim, bias=False) for split_dim in split_dims]) + for lora_down in self.lora_down: + torch.nn.init.kaiming_uniform_(lora_down.weight, a=math.sqrt(5)) + for lora_up in self.lora_up: + torch.nn.init.zeros_(lora_up.weight) + + if type(alpha) == torch.Tensor: + alpha = alpha.detach().float().numpy() # without casting, bf16 causes error + alpha = self.lora_dim if alpha is None or alpha == 0 else alpha + self.scale = alpha / self.lora_dim + self.register_buffer("alpha", torch.tensor(alpha)) # 定数として扱える + + # same as microsoft's + self.multiplier = multiplier + self.org_module = org_module # remove in applying + self.dropout = dropout + self.rank_dropout = rank_dropout + self.module_dropout = module_dropout + + self.ggpo_sigma = ggpo_sigma + self.ggpo_beta = ggpo_beta + + if self.ggpo_beta is not None and self.ggpo_sigma is not None: + self.combined_weight_norms = None + self.grad_norms = None + self.perturbation_norm_factor = 1.0 / math.sqrt(org_module.weight.shape[0]) + self.initialize_norm_cache(org_module.weight) + self.org_module_shape: tuple[int] = org_module.weight.shape + + def apply_to(self): + self.org_forward = self.org_module.forward + self.org_module.forward = self.forward + + del self.org_module + + def forward(self, x): + org_forwarded = self.org_forward(x) + + # module dropout + if self.module_dropout is not None and self.training: + if torch.rand(1) < self.module_dropout: + return org_forwarded + + if self.split_dims is None: + lx = self.lora_down(x) + + # normal dropout + if self.dropout is not None and self.training: + lx = torch.nn.functional.dropout(lx, p=self.dropout) + + # rank dropout + if self.rank_dropout is not None and self.training: + mask = torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout + if len(lx.size()) == 3: + mask = mask.unsqueeze(1) # for Text Encoder + elif len(lx.size()) == 4: + mask = mask.unsqueeze(-1).unsqueeze(-1) # for Conv2d + lx = lx * mask + + # scaling for rank dropout: treat as if the rank is changed + # maskから計算することも考えられるが、augmentation的な効果を期待してrank_dropoutを用いる + scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability + else: + scale = self.scale + + lx = self.lora_up(lx) + + # LoRA Gradient-Guided Perturbation Optimization + if self.training and self.ggpo_sigma is not None and self.ggpo_beta is not None and self.combined_weight_norms is not None and self.grad_norms is not None: + with torch.no_grad(): + perturbation_scale = (self.ggpo_sigma * torch.sqrt(self.combined_weight_norms ** 2)) + (self.ggpo_beta * (self.grad_norms ** 2)) + perturbation_scale_factor = (perturbation_scale * self.perturbation_norm_factor).to(self.device) + perturbation = torch.randn(self.org_module_shape, dtype=self.dtype, device=self.device) + perturbation.mul_(perturbation_scale_factor) + perturbation_output = x @ perturbation.T # Result: (batch × n) + return org_forwarded + (self.multiplier * scale * lx) + perturbation_output + else: + return org_forwarded + lx * self.multiplier * scale + else: + lxs = [lora_down(x) for lora_down in self.lora_down] + + # normal dropout + if self.dropout is not None and self.training: + lxs = [torch.nn.functional.dropout(lx, p=self.dropout) for lx in lxs] + + # rank dropout + if self.rank_dropout is not None and self.training: + masks = [torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout for lx in lxs] + for i in range(len(lxs)): + if len(lx.size()) == 3: + masks[i] = masks[i].unsqueeze(1) + elif len(lx.size()) == 4: + masks[i] = masks[i].unsqueeze(-1).unsqueeze(-1) + lxs[i] = lxs[i] * masks[i] + + # scaling for rank dropout: treat as if the rank is changed + scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability + else: + scale = self.scale + + lxs = [lora_up(lx) for lora_up, lx in zip(self.lora_up, lxs)] + + return org_forwarded + torch.cat(lxs, dim=-1) * self.multiplier * scale + + @torch.no_grad() + def initialize_norm_cache(self, org_module_weight: Tensor): + # Choose a reasonable sample size + n_rows = org_module_weight.shape[0] + sample_size = min(1000, n_rows) # Cap at 1000 samples or use all if smaller + + # Sample random indices across all rows + indices = torch.randperm(n_rows)[:sample_size] + + # Convert to a supported data type first, then index + # Use float32 for indexing operations + weights_float32 = org_module_weight.to(dtype=torch.float32) + sampled_weights = weights_float32[indices].to(device=self.device) + + # Calculate sampled norms + sampled_norms = torch.norm(sampled_weights, dim=1, keepdim=True) + + # Store the mean norm as our estimate + self.org_weight_norm_estimate = sampled_norms.mean() + + # Optional: store standard deviation for confidence intervals + self.org_weight_norm_std = sampled_norms.std() + + # Free memory + del sampled_weights, weights_float32 + + @torch.no_grad() + def validate_norm_approximation(self, org_module_weight: Tensor, verbose=True): + # Calculate the true norm (this will be slow but it's just for validation) + true_norms = [] + chunk_size = 1024 # Process in chunks to avoid OOM + + for i in range(0, org_module_weight.shape[0], chunk_size): + end_idx = min(i + chunk_size, org_module_weight.shape[0]) + chunk = org_module_weight[i:end_idx].to(device=self.device, dtype=self.dtype) + chunk_norms = torch.norm(chunk, dim=1, keepdim=True) + true_norms.append(chunk_norms.cpu()) + del chunk + + true_norms = torch.cat(true_norms, dim=0) + true_mean_norm = true_norms.mean().item() + + # Compare with our estimate + estimated_norm = self.org_weight_norm_estimate.item() + + # Calculate error metrics + absolute_error = abs(true_mean_norm - estimated_norm) + relative_error = absolute_error / true_mean_norm * 100 # as percentage + + if verbose: + logger.info(f"True mean norm: {true_mean_norm:.6f}") + logger.info(f"Estimated norm: {estimated_norm:.6f}") + logger.info(f"Absolute error: {absolute_error:.6f}") + logger.info(f"Relative error: {relative_error:.2f}%") + + return { + 'true_mean_norm': true_mean_norm, + 'estimated_norm': estimated_norm, + 'absolute_error': absolute_error, + 'relative_error': relative_error + } + + + @torch.no_grad() + def update_norms(self): + # Not running GGPO so not currently running update norms + if self.ggpo_beta is None or self.ggpo_sigma is None: + return + + # only update norms when we are training + if self.training is False: + return + + module_weights = self.lora_up.weight @ self.lora_down.weight + module_weights.mul(self.scale) + + self.weight_norms = torch.norm(module_weights, dim=1, keepdim=True) + self.combined_weight_norms = torch.sqrt((self.org_weight_norm_estimate**2) + + torch.sum(module_weights**2, dim=1, keepdim=True)) + + @torch.no_grad() + def update_grad_norms(self): + if self.training is False: + print(f"skipping update_grad_norms for {self.lora_name}") + return + + lora_down_grad = None + lora_up_grad = None + + for name, param in self.named_parameters(): + if name == "lora_down.weight": + lora_down_grad = param.grad + elif name == "lora_up.weight": + lora_up_grad = param.grad + + # Calculate gradient norms if we have both gradients + if lora_down_grad is not None and lora_up_grad is not None: + with torch.autocast(self.device.type): + approx_grad = self.scale * ((self.lora_up.weight @ lora_down_grad) + (lora_up_grad @ self.lora_down.weight)) + self.grad_norms = torch.norm(approx_grad, dim=1, keepdim=True) + + + @property + def device(self): + return next(self.parameters()).device + + @property + def dtype(self): + return next(self.parameters()).dtype + + +class LoRAInfModule(LoRAModule): + def __init__( + self, + lora_name, + org_module: torch.nn.Module, + multiplier=1.0, + lora_dim=4, + alpha=1, + **kwargs, + ): + # no dropout for inference + super().__init__(lora_name, org_module, multiplier, lora_dim, alpha) + + self.org_module_ref = [org_module] # 後から参照できるように + self.enabled = True + self.network: LoRANetwork = None + + def set_network(self, network): + self.network = network + + # freezeしてマージする + def merge_to(self, sd, dtype, device): + # extract weight from org_module + org_sd = self.org_module.state_dict() + weight = org_sd["weight"] + org_dtype = weight.dtype + org_device = weight.device + weight = weight.to(torch.float) # calc in float + + if dtype is None: + dtype = org_dtype + if device is None: + device = org_device + + if self.split_dims is None: + # get up/down weight + down_weight = sd["lora_down.weight"].to(torch.float).to(device) + up_weight = sd["lora_up.weight"].to(torch.float).to(device) + + # merge weight + if len(weight.size()) == 2: + # linear + weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale + elif down_weight.size()[2:4] == (1, 1): + # conv2d 1x1 + weight = ( + weight + + self.multiplier + * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3) + * self.scale + ) + else: + # conv2d 3x3 + conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3) + # logger.info(conved.size(), weight.size(), module.stride, module.padding) + weight = weight + self.multiplier * conved * self.scale + + # set weight to org_module + org_sd["weight"] = weight.to(dtype) + self.org_module.load_state_dict(org_sd) + else: + # split_dims + total_dims = sum(self.split_dims) + for i in range(len(self.split_dims)): + # get up/down weight + down_weight = sd[f"lora_down.{i}.weight"].to(torch.float).to(device) # (rank, in_dim) + up_weight = sd[f"lora_up.{i}.weight"].to(torch.float).to(device) # (split dim, rank) + + # pad up_weight -> (total_dims, rank) + padded_up_weight = torch.zeros((total_dims, up_weight.size(0)), device=device, dtype=torch.float) + padded_up_weight[sum(self.split_dims[:i]) : sum(self.split_dims[: i + 1])] = up_weight + + # merge weight + weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale + + # set weight to org_module + org_sd["weight"] = weight.to(dtype) + self.org_module.load_state_dict(org_sd) + + # 復元できるマージのため、このモジュールのweightを返す + def get_weight(self, multiplier=None): + if multiplier is None: + multiplier = self.multiplier + + # get up/down weight from module + up_weight = self.lora_up.weight.to(torch.float) + down_weight = self.lora_down.weight.to(torch.float) + + # pre-calculated weight + if len(down_weight.size()) == 2: + # linear + weight = self.multiplier * (up_weight @ down_weight) * self.scale + elif down_weight.size()[2:4] == (1, 1): + # conv2d 1x1 + weight = ( + self.multiplier + * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3) + * self.scale + ) + else: + # conv2d 3x3 + conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3) + weight = self.multiplier * conved * self.scale + + return weight + + def set_region(self, region): + self.region = region + self.region_mask = None + + def default_forward(self, x): + # logger.info(f"default_forward {self.lora_name} {x.size()}") + if self.split_dims is None: + lx = self.lora_down(x) + lx = self.lora_up(lx) + return self.org_forward(x) + lx * self.multiplier * self.scale + else: + lxs = [lora_down(x) for lora_down in self.lora_down] + lxs = [lora_up(lx) for lora_up, lx in zip(self.lora_up, lxs)] + return self.org_forward(x) + torch.cat(lxs, dim=-1) * self.multiplier * self.scale + + def forward(self, x): + if not self.enabled: + return self.org_forward(x) + return self.default_forward(x) + + +def create_network( + multiplier: float, + network_dim: Optional[int],#LoRA 的秩（rank），决定 LoRA 模块的参数量。64 + network_alpha: Optional[float],# alpha / dim 是缩放比例 32 + ae: AutoencoderKL, + text_encoders, + base_dit, + neuron_dropout: Optional[float] = None, + **kwargs, +): + if network_dim is None: + network_dim = 4 # default + if network_alpha is None: + network_alpha = 1.0 + + # extract dim/alpha for conv2d, and block dim + conv_dim = kwargs.get("conv_dim", None) + conv_alpha = kwargs.get("conv_alpha", None) + if conv_dim is not None: + conv_dim = int(conv_dim) + if conv_alpha is None: + conv_alpha = 1.0 + else: + conv_alpha = float(conv_alpha) + + # attn dim, mlp dim: only for DoubleStreamBlock. SingleStreamBlock is not supported because of combined qkv 用于为 DiT 模型中不同类型的模块（图像/文本注意力、MLP、调制层，以及单流/双流块）指定不同的 LoRA 秩。这些存储在 type_dims 列表中 + img_attn_dim = kwargs.get("img_attn_dim", None) + txt_attn_dim = kwargs.get("txt_attn_dim", None) + img_mlp_dim = kwargs.get("img_mlp_dim", None) + txt_mlp_dim = kwargs.get("txt_mlp_dim", None) + img_mod_dim = kwargs.get("img_mod_dim", None) + txt_mod_dim = kwargs.get("txt_mod_dim", None) + single_dim = kwargs.get("single_dim", None) # SingleStreamBlock + single_mod_dim = kwargs.get("single_mod_dim", None) # SingleStreamBlock + if img_attn_dim is not None: + img_attn_dim = int(img_attn_dim) + if txt_attn_dim is not None: + txt_attn_dim = int(txt_attn_dim) + if img_mlp_dim is not None: + img_mlp_dim = int(img_mlp_dim) + if txt_mlp_dim is not None: + txt_mlp_dim = int(txt_mlp_dim) + if img_mod_dim is not None: + img_mod_dim = int(img_mod_dim) + if txt_mod_dim is not None: + txt_mod_dim = int(txt_mod_dim) + if single_dim is not None: + single_dim = int(single_dim) + if single_mod_dim is not None: + single_mod_dim = int(single_mod_dim) + type_dims = [img_attn_dim, txt_attn_dim, img_mlp_dim, txt_mlp_dim, img_mod_dim, txt_mod_dim, single_dim, single_mod_dim] + if all([d is None for d in type_dims]): + type_dims = None + + # in_dims [img, time, vector, guidance, txt]用于指定输入层（图像、时间、向量、引导、文本）的 LoRA 秩 + in_dims = kwargs.get("in_dims", None) + if in_dims is not None: + in_dims = in_dims.strip() + if in_dims.startswith("[") and in_dims.endswith("]"): + in_dims = in_dims[1:-1] + in_dims = [int(d) for d in in_dims.split(",")] # is it better to use ast.literal_eval? + assert len(in_dims) == 5, f"invalid in_dims: {in_dims}, must be 5 dimensions (img, time, vector, guidance, txt)" + + # double/single train blocks + def parse_block_selection(selection: str, total_blocks: int) -> List[bool]: + """ + Parse a block selection string and return a list of booleans. + + Args: + selection (str): A string specifying which blocks to select. + total_blocks (int): The total number of blocks available. + + Returns: + List[bool]: A list of booleans indicating which blocks are selected. + """ + if selection == "all": + return [True] * total_blocks + if selection == "none" or selection == "": + return [False] * total_blocks + + selected = [False] * total_blocks + ranges = selection.split(",") + + for r in ranges: + if "-" in r: + start, end = map(str.strip, r.split("-")) + start = int(start) + end = int(end) + assert 0 <= start < total_blocks, f"invalid start index: {start}" + assert 0 <= end < total_blocks, f"invalid end index: {end}" + assert start <= end, f"invalid range: {start}-{end}" + for i in range(start, end + 1): + selected[i] = True + else: + index = int(r) + assert 0 <= index < total_blocks, f"invalid index: {index}" + selected[index] = True + + return selected + + train_double_block_indices = kwargs.get("train_double_block_indices", None) + train_single_block_indices = kwargs.get("train_single_block_indices", None) + if train_double_block_indices is not None: + train_double_block_indices = parse_block_selection(train_double_block_indices, NUM_DOUBLE_BLOCKS) + if train_single_block_indices is not None: + train_single_block_indices = parse_block_selection(train_single_block_indices, NUM_SINGLE_BLOCKS) + + # rank/module dropout + rank_dropout = kwargs.get("rank_dropout", None) + if rank_dropout is not None: + rank_dropout = float(rank_dropout) + module_dropout = kwargs.get("module_dropout", None) + if module_dropout is not None: + module_dropout = float(module_dropout) + + # single or double blocks + train_blocks = kwargs.get("train_blocks", None) # None (default), "all" (same as None), "single", "double"指定只训练 "all" (所有，默认), "single" (只训练单流块) 或 "double" (只训练双流块) + if train_blocks is not None: + assert train_blocks in ["all", "single", "double"], f"invalid train_blocks: {train_blocks}" + + # split qkv + split_qkv = kwargs.get("split_qkv", False)#是否将 qkv 矩阵拆分为单独的权重 + if split_qkv is not None: + split_qkv = True if split_qkv == "True" else False + + ggpo_beta = kwargs.get("ggpo_beta", None) + ggpo_sigma = kwargs.get("ggpo_sigma", None)#与 LoRA Gradient-Guided Perturbation Optimization (GGPO) 训练策略相关的参数 + + if ggpo_beta is not None: + ggpo_beta = float(ggpo_beta) + + if ggpo_sigma is not None: + ggpo_sigma = float(ggpo_sigma) + + + train_qwen = kwargs.get("train_qwen", False) + if train_qwen is not None: + train_qwen = True if train_qwen == "True" else False + + # verbose + verbose = kwargs.get("verbose", False) + if verbose is not None: + verbose = True if verbose == "True" else False + + # すごく引数が多いな ( ^ω^)･･･ + network = LoRANetwork( + text_encoders, + base_dit, + multiplier=multiplier, + lora_dim=network_dim, + alpha=network_alpha, + dropout=neuron_dropout, + rank_dropout=rank_dropout, + module_dropout=module_dropout, + conv_lora_dim=conv_dim, + conv_alpha=conv_alpha, + train_blocks=train_blocks, + split_qkv=split_qkv, + train_qwen=train_qwen, + type_dims=type_dims, + in_dims=in_dims, + train_double_block_indices=train_double_block_indices, + train_single_block_indices=train_single_block_indices, + ggpo_beta=ggpo_beta, + ggpo_sigma=ggpo_sigma, + verbose=verbose, + ) + # 用于设置 LoRA+ 的训练参数，学习率比例参数 + loraplus_lr_ratio = kwargs.get("loraplus_lr_ratio", None) + loraplus_unet_lr_ratio = kwargs.get("loraplus_unet_lr_ratio", None) + loraplus_text_encoder_lr_ratio = kwargs.get("loraplus_text_encoder_lr_ratio", None) + loraplus_lr_ratio = float(loraplus_lr_ratio) if loraplus_lr_ratio is not None else None + loraplus_unet_lr_ratio = float(loraplus_unet_lr_ratio) if loraplus_unet_lr_ratio is not None else None + loraplus_text_encoder_lr_ratio = float(loraplus_text_encoder_lr_ratio) if loraplus_text_encoder_lr_ratio is not None else None + if loraplus_lr_ratio is not None or loraplus_unet_lr_ratio is not None or loraplus_text_encoder_lr_ratio is not None: + network.set_loraplus_lr_ratio(loraplus_lr_ratio, loraplus_unet_lr_ratio, loraplus_text_encoder_lr_ratio) + + return network + + +# Create network from weights for inference, weights are not loaded here (because can be merged) +def create_network_from_weights(multiplier, file, ae, text_encoders, base_dit, weights_sd=None, for_inference=False, **kwargs): + if weights_sd is None: + if os.path.splitext(file)[1] == ".safetensors": + from safetensors.torch import load_file, safe_open + + weights_sd = load_file(file) + else: + weights_sd = torch.load(file, map_location="cpu") + + modules_dim = {} + modules_alpha = {} + train_qwen = None + for key, value in weights_sd.items(): + if "." not in key: + continue + + lora_name = key.split(".")[0] + if "alpha" in key: + modules_alpha[lora_name] = value + elif "lora_down" in key: + dim = value.size()[0] + modules_dim[lora_name] = dim + # logger.info(lora_name, value.size(), dim) + + if train_qwen is None or train_qwen is False: + train_qwen = "lora_te3" in lora_name + + if train_qwen is None: + train_qwen = False + + split_qkv = False # split_qkv is not needed to care, because state_dict is qkv combined + + module_class = LoRAInfModule if for_inference else LoRAModule + + network = LoRANetwork( + text_encoders, + base_dit, + multiplier=multiplier, + modules_dim=modules_dim, + modules_alpha=modules_alpha, + module_class=module_class, + split_qkv=split_qkv, + train_qwen=train_qwen, + ) + return network, weights_sd + + +class LoRANetwork(torch.nn.Module): + DIT_TARGET_REPLACE_MODULE_DOUBLE = ["DoubleStreamBlock"] + DIT_TARGET_REPLACE_MODULE_SINGLE = ["SingleStreamBlock"] + TEXT_ENCODER_TARGET_REPLACE_MODULE = ["Qwen2MLP", "Qwen2_5_VLAttention"] + LORA_PREFIX_DIT = "lora_unet" # make ComfyUI compatible + LORA_PREFIX_TEXT_ENCODER = "lora_te" # make ComfyUI compatible + + def __init__( + self, + text_encoders, + unet, + multiplier: float = 1.0, + lora_dim: int = 4, + alpha: float = 1, + dropout: Optional[float] = None, + rank_dropout: Optional[float] = None, + module_dropout: Optional[float] = None, + conv_lora_dim: Optional[int] = None, + conv_alpha: Optional[float] = None, + module_class: Type[object] = LoRAModule, + modules_dim: Optional[Dict[str, int]] = None, + modules_alpha: Optional[Dict[str, int]] = None, + train_blocks: Optional[str] = None, + split_qkv: bool = False, + train_qwen: bool = False, + type_dims: Optional[List[int]] = None, + in_dims: Optional[List[int]] = None, + train_double_block_indices: Optional[List[bool]] = None, + train_single_block_indices: Optional[List[bool]] = None, + ggpo_beta: Optional[float] = None, + ggpo_sigma: Optional[float] = None, + verbose: Optional[bool] = False, + ) -> None: + super().__init__() + self.multiplier = multiplier + + self.lora_dim = lora_dim + self.alpha = alpha + self.conv_lora_dim = conv_lora_dim + self.conv_alpha = conv_alpha + self.dropout = dropout + self.rank_dropout = rank_dropout + self.module_dropout = module_dropout + self.train_blocks = train_blocks if train_blocks is not None else "all" + self.split_qkv = split_qkv + self.train_qwen = train_qwen + + self.type_dims = type_dims + self.in_dims = in_dims + self.train_double_block_indices = train_double_block_indices + self.train_single_block_indices = train_single_block_indices + + self.loraplus_lr_ratio = None + self.loraplus_unet_lr_ratio = None + self.loraplus_text_encoder_lr_ratio = None + + if modules_dim is not None: + logger.info(f"create LoRA network from weights") + self.in_dims = [0] * 5 # create in_dims + # verbose = True + else: + logger.info(f"create LoRA network. base dim (rank): {lora_dim}, alpha: {alpha}") + logger.info( + f"neuron dropout: p={self.dropout}, rank dropout: p={self.rank_dropout}, module dropout: p={self.module_dropout}" + ) + + if ggpo_beta is not None and ggpo_sigma is not None: + logger.info(f"LoRA-GGPO training sigma: {ggpo_sigma} beta: {ggpo_beta}") + + if self.split_qkv: + logger.info(f"split qkv for LoRA") + if self.train_blocks is not None: + logger.info(f"train {self.train_blocks} blocks only") + + + if train_qwen: + logger.info(f"train qwen as well") + + # create module instances + def create_modules( + is_dit: bool, + text_encoder_idx: Optional[int], + root_module: torch.nn.Module, + target_replace_modules: List[str], + filter: Optional[str] = None, + default_dim: Optional[int] = None, + ) -> List[LoRAModule]: + prefix = ( + self.LORA_PREFIX_DIT + if is_dit + else self.LORA_PREFIX_TEXT_ENCODER + ) + + loras = [] + skipped = [] + for name, module in root_module.named_modules(): + if target_replace_modules is None or module.__class__.__name__ in target_replace_modules: + if target_replace_modules is None: # dirty hack for all modules + module = root_module # search all modules + + for child_name, child_module in module.named_modules(): + is_linear = child_module.__class__.__name__ == "Linear" + is_conv2d = child_module.__class__.__name__ == "Conv2d" + is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1) + + if is_linear or is_conv2d: + lora_name = prefix + "." + (name + "." if name else "") + child_name + lora_name = lora_name.replace(".", "_") + + if filter is not None and not filter in lora_name: + continue + + dim = None + alpha = None + + if modules_dim is not None: + # モジュール指定あり + if lora_name in modules_dim: + dim = modules_dim[lora_name] + alpha = modules_alpha[lora_name] + else: + # 通常、すべて対象とする + if is_linear or is_conv2d_1x1: + dim = default_dim if default_dim is not None else self.lora_dim + alpha = self.alpha + + if is_dit and type_dims is not None: + identifier = [ + ("img_attn",), + ("txt_attn",), + ("img_mlp",), + ("txt_mlp",), + ("img_mod",), + ("txt_mod",), + ("single_blocks", "linear"), + ("modulation",), + ] + for i, d in enumerate(type_dims): + if d is not None and all([id in lora_name for id in identifier[i]]): + dim = d # may be 0 for skip + break + + if ( + is_dit + and dim + and ( + self.train_double_block_indices is not None + or self.train_single_block_indices is not None + ) + and ("double" in lora_name or "single" in lora_name) + ): + # "lora_unet_double_blocks_0_..." or "lora_unet_single_blocks_0_..." + block_index = int(lora_name.split("_")[4]) # bit dirty + if ( + "double" in lora_name + and self.train_double_block_indices is not None + and not self.train_double_block_indices[block_index] + ): + dim = 0 + elif ( + "single" in lora_name + and self.train_single_block_indices is not None + and not self.train_single_block_indices[block_index] + ): + dim = 0 + + elif self.conv_lora_dim is not None: + dim = self.conv_lora_dim + alpha = self.conv_alpha + + if dim is None or dim == 0: + # skipした情報を出力 + if is_linear or is_conv2d_1x1 or (self.conv_lora_dim is not None): + skipped.append(lora_name) + continue + + # qkv split + split_dims = None + if is_dit and split_qkv: + if "double" in lora_name and "qkv" in lora_name: + split_dims = [3072] * 3 + elif "single" in lora_name and "linear1" in lora_name: + split_dims = [3072] * 3 + [12288] + + lora = module_class( + lora_name, + child_module, + self.multiplier, + dim, + alpha, + dropout=dropout, + rank_dropout=rank_dropout, + module_dropout=module_dropout, + split_dims=split_dims, + ggpo_beta=ggpo_beta, + ggpo_sigma=ggpo_sigma, + ) + loras.append(lora) + + if target_replace_modules is None: + break # all modules are searched + return loras, skipped + + # create LoRA for text encoder + # 毎回すべてのモジュールを作るのは無駄なので要検討 + self.text_encoder_loras: List[Union[LoRAModule, LoRAInfModule]] = [] + skipped_te = [] + for i, text_encoder in enumerate(text_encoders): + index = i + if not train_qwen: + break + + logger.info(f"create LoRA for Text Encoder {index+1}:") + + text_encoder_loras, skipped = create_modules(False, index, text_encoder, LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE) + logger.info(f"create LoRA for Text Encoder {index+1}: {len(text_encoder_loras)} modules.") + self.text_encoder_loras.extend(text_encoder_loras) + skipped_te += skipped + + # create LoRA for U-Net + if self.train_blocks == "all": + target_replace_modules = LoRANetwork.DIT_TARGET_REPLACE_MODULE_DOUBLE + LoRANetwork.DIT_TARGET_REPLACE_MODULE_SINGLE + elif self.train_blocks == "single": + target_replace_modules = LoRANetwork.DIT_TARGET_REPLACE_MODULE_SINGLE + elif self.train_blocks == "double": + target_replace_modules = LoRANetwork.DIT_TARGET_REPLACE_MODULE_DOUBLE + + self.unet_loras: List[Union[LoRAModule, LoRAInfModule]] + self.unet_loras, skipped_un = create_modules(True, None, unet, target_replace_modules) + + # img, time, vector, guidance, txt + if self.in_dims: + for filter, in_dim in zip(["_img_in", "_time_in", "_vector_in", "_guidance_in", "_txt_in"], self.in_dims): + loras, _ = create_modules(True, None, unet, None, filter=filter, default_dim=in_dim) + self.unet_loras.extend(loras) + + logger.info(f"create LoRA for DIT {self.train_blocks} blocks: {len(self.unet_loras)} modules.") + if verbose: + for lora in self.unet_loras: + logger.info(f"\t{lora.lora_name:50} {lora.lora_dim}, {lora.alpha}") + + skipped = skipped_te + skipped_un + if verbose and len(skipped) > 0: + logger.warning( + f"because dim (rank) is 0, {len(skipped)} LoRA modules are skipped / dim (rank)が0の為、次の{len(skipped)}個のLoRAモジュールはスキップされます:" + ) + for name in skipped: + logger.info(f"\t{name}") + + # assertion + names = set() + for lora in self.text_encoder_loras + self.unet_loras: + assert lora.lora_name not in names, f"duplicated lora name: {lora.lora_name}" + names.add(lora.lora_name) + + def set_multiplier(self, multiplier): + self.multiplier = multiplier + for lora in self.text_encoder_loras + self.unet_loras: + lora.multiplier = self.multiplier + + def set_enabled(self, is_enabled): + for lora in self.text_encoder_loras + self.unet_loras: + lora.enabled = is_enabled + + def update_norms(self): + for lora in self.text_encoder_loras + self.unet_loras: + lora.update_norms() + + def update_grad_norms(self): + for lora in self.text_encoder_loras + self.unet_loras: + lora.update_grad_norms() + + def grad_norms(self) -> Tensor | None: + grad_norms = [] + for lora in self.text_encoder_loras + self.unet_loras: + if hasattr(lora, "grad_norms") and lora.grad_norms is not None: + grad_norms.append(lora.grad_norms.mean(dim=0)) + return torch.stack(grad_norms) if len(grad_norms) > 0 else None + + def weight_norms(self) -> Tensor | None: + weight_norms = [] + for lora in self.text_encoder_loras + self.unet_loras: + if hasattr(lora, "weight_norms") and lora.weight_norms is not None: + weight_norms.append(lora.weight_norms.mean(dim=0)) + return torch.stack(weight_norms) if len(weight_norms) > 0 else None + + def combined_weight_norms(self) -> Tensor | None: + combined_weight_norms = [] + for lora in self.text_encoder_loras + self.unet_loras: + if hasattr(lora, "combined_weight_norms") and lora.combined_weight_norms is not None: + combined_weight_norms.append(lora.combined_weight_norms.mean(dim=0)) + return torch.stack(combined_weight_norms) if len(combined_weight_norms) > 0 else None + + + def load_weights(self, file): + if os.path.splitext(file)[1] == ".safetensors": + from safetensors.torch import load_file + + weights_sd = load_file(file) + else: + weights_sd = torch.load(file, map_location="cpu") + + info = self.load_state_dict(weights_sd, False) + return info + + def load_state_dict(self, state_dict, strict=True): + # override to convert original weight to split qkv + if not self.split_qkv: + return super().load_state_dict(state_dict, strict) + + # split qkv + for key in list(state_dict.keys()): + if "double" in key and "qkv" in key: + split_dims = [3072] * 3 + elif "single" in key and "linear1" in key: + split_dims = [3072] * 3 + [12288] + else: + continue + + weight = state_dict[key] + lora_name = key.split(".")[0] + if "lora_down" in key and "weight" in key: + # dense weight (rank*3, in_dim) + split_weight = torch.chunk(weight, len(split_dims), dim=0) + for i, split_w in enumerate(split_weight): + state_dict[f"{lora_name}.lora_down.{i}.weight"] = split_w + + del state_dict[key] + # print(f"split {key}: {weight.shape} to {[w.shape for w in split_weight]}") + elif "lora_up" in key and "weight" in key: + # sparse weight (out_dim=sum(split_dims), rank*3) + rank = weight.size(1) // len(split_dims) + i = 0 + for j in range(len(split_dims)): + state_dict[f"{lora_name}.lora_up.{j}.weight"] = weight[i : i + split_dims[j], j * rank : (j + 1) * rank] + i += split_dims[j] + del state_dict[key] + + + return super().load_state_dict(state_dict, strict) + + def state_dict(self, destination=None, prefix="", keep_vars=False): + if not self.split_qkv: + return super().state_dict(destination, prefix, keep_vars) + + # merge qkv + state_dict = super().state_dict(destination, prefix, keep_vars) + new_state_dict = {} + for key in list(state_dict.keys()): + if "double" in key and "qkv" in key: + split_dims = [3072] * 3 + elif "single" in key and "linear1" in key: + split_dims = [3072] * 3 + [12288] + else: + new_state_dict[key] = state_dict[key] + continue + + if key not in state_dict: + continue # already merged + + lora_name = key.split(".")[0] + + # (rank, in_dim) * 3 + down_weights = [state_dict.pop(f"{lora_name}.lora_down.{i}.weight") for i in range(len(split_dims))] + # (split dim, rank) * 3 + up_weights = [state_dict.pop(f"{lora_name}.lora_up.{i}.weight") for i in range(len(split_dims))] + + alpha = state_dict.pop(f"{lora_name}.alpha") + + # merge down weight + down_weight = torch.cat(down_weights, dim=0) # (rank, split_dim) * 3 -> (rank*3, sum of split_dim) + + # merge up weight (sum of split_dim, rank*3) + rank = up_weights[0].size(1) + up_weight = torch.zeros((sum(split_dims), down_weight.size(0)), device=down_weight.device, dtype=down_weight.dtype) + i = 0 + for j in range(len(split_dims)): + up_weight[i : i + split_dims[j], j * rank : (j + 1) * rank] = up_weights[j] + i += split_dims[j] + + new_state_dict[f"{lora_name}.lora_down.weight"] = down_weight + new_state_dict[f"{lora_name}.lora_up.weight"] = up_weight + new_state_dict[f"{lora_name}.alpha"] = alpha + + # print( + # f"merged {lora_name}: {lora_name}, {[w.shape for w in down_weights]}, {[w.shape for w in up_weights]} to {down_weight.shape}, {up_weight.shape}" + # ) + print(f"new key: {lora_name}.lora_down.weight, {lora_name}.lora_up.weight, {lora_name}.alpha") + + return new_state_dict + + def apply_to(self, text_encoders, dit, apply_text_encoder=True, apply_unet=True): + if apply_text_encoder: + logger.info(f"enable LoRA for text encoder: {len(self.text_encoder_loras)} modules") + else: + self.text_encoder_loras = [] + + if apply_unet: + logger.info(f"enable LoRA for U-Net: {len(self.unet_loras)} modules") + else: + self.unet_loras = [] + + for lora in self.text_encoder_loras + self.unet_loras: + lora.apply_to() + self.add_module(lora.lora_name, lora) + + # マージできるかどうかを返す + def is_mergeable(self): + return True + + # TODO refactor to common function with apply_to + def merge_to(self, text_encoders, dit, weights_sd, dtype=None, device=None): + apply_text_encoder = apply_unet = False + for key in weights_sd.keys(): + if key.startswith(LoRANetwork.LORA_PREFIX_TEXT_ENCODER): + apply_text_encoder = True + elif key.startswith(LoRANetwork.LORA_PREFIX_DIT): + apply_unet = True + + if apply_text_encoder: + logger.info("enable LoRA for text encoder") + else: + self.text_encoder_loras = [] + + if apply_unet: + logger.info("enable LoRA for U-Net") + else: + self.unet_loras = [] + + for lora in self.text_encoder_loras + self.unet_loras: + sd_for_lora = {} + for key in weights_sd.keys(): + if key.startswith(lora.lora_name): + sd_for_lora[key[len(lora.lora_name) + 1 :]] = weights_sd[key] + lora.merge_to(sd_for_lora, dtype, device) + + logger.info(f"weights are merged") + + def set_loraplus_lr_ratio(self, loraplus_lr_ratio, loraplus_unet_lr_ratio, loraplus_text_encoder_lr_ratio): + self.loraplus_lr_ratio = loraplus_lr_ratio + self.loraplus_unet_lr_ratio = loraplus_unet_lr_ratio + self.loraplus_text_encoder_lr_ratio = loraplus_text_encoder_lr_ratio + + logger.info(f"LoRA+ UNet LR Ratio: {self.loraplus_unet_lr_ratio or self.loraplus_lr_ratio}") + logger.info(f"LoRA+ Text Encoder LR Ratio: {self.loraplus_text_encoder_lr_ratio or self.loraplus_lr_ratio}") + + def prepare_optimizer_params_with_multiple_te_lrs(self, text_encoder_lr, unet_lr, default_lr): + # make sure text_encoder_lr as list of two elements + # if float, use the same value for both text encoders + if text_encoder_lr is None or (isinstance(text_encoder_lr, list) and len(text_encoder_lr) == 0): + text_encoder_lr = [default_lr, default_lr] + elif isinstance(text_encoder_lr, float) or isinstance(text_encoder_lr, int): + text_encoder_lr = [float(text_encoder_lr), float(text_encoder_lr)] + elif len(text_encoder_lr) == 1: + text_encoder_lr = [text_encoder_lr[0], text_encoder_lr[0]] + + self.requires_grad_(True) + + all_params = [] + lr_descriptions = [] + + def assemble_params(loras, lr, loraplus_ratio): + param_groups = {"lora": {}, "plus": {}} + for lora in loras: + for name, param in lora.named_parameters(): + if loraplus_ratio is not None and "lora_up" in name: + param_groups["plus"][f"{lora.lora_name}.{name}"] = param + else: + param_groups["lora"][f"{lora.lora_name}.{name}"] = param + + params = [] + descriptions = [] + for key in param_groups.keys(): + param_data = {"params": param_groups[key].values()} + + if len(param_data["params"]) == 0: + continue + + if lr is not None: + if key == "plus": + param_data["lr"] = lr * loraplus_ratio + else: + param_data["lr"] = lr + + if param_data.get("lr", None) == 0 or param_data.get("lr", None) is None: + logger.info("NO LR skipping!") + continue + + params.append(param_data) + descriptions.append("plus" if key == "plus" else "") + + return params, descriptions + + if self.text_encoder_loras: + loraplus_lr_ratio = self.loraplus_text_encoder_lr_ratio or self.loraplus_lr_ratio + + # split text encoder loras for te1 and te3 + te_loras = [lora for lora in self.text_encoder_loras if lora.lora_name.startswith(self.LORA_PREFIX_TEXT_ENCODER)] + if len(te_loras) > 0: + logger.info(f"Text Encoder: {len(te_loras)} modules, LR {text_encoder_lr[0]}") + params, descriptions = assemble_params(te_loras, text_encoder_lr[0], loraplus_lr_ratio) + all_params.extend(params) + lr_descriptions.extend(["textencoder" + (" " + d if d else "") for d in descriptions]) + + if self.unet_loras: + params, descriptions = assemble_params( + self.unet_loras, + unet_lr if unet_lr is not None else default_lr, + self.loraplus_unet_lr_ratio or self.loraplus_lr_ratio, + ) + all_params.extend(params) + lr_descriptions.extend(["unet" + (" " + d if d else "") for d in descriptions]) + + return all_params, lr_descriptions + + def enable_gradient_checkpointing(self): + # not supported + pass + + def prepare_grad_etc(self, text_encoder, unet): + self.requires_grad_(True) + + def on_epoch_start(self, text_encoder, unet): + self.train() + + def get_trainable_params(self): + return self.parameters() + + def save_weights(self, file, dtype, metadata=None): + if metadata is not None and len(metadata) == 0: + metadata = None + + state_dict = self.state_dict() + + if dtype is not None: + for key in list(state_dict.keys()): + v = state_dict[key] + v = v.detach().clone().to("cpu").to(dtype) + state_dict[key] = v + + if os.path.splitext(file)[1] == ".safetensors": + from safetensors.torch import save_file + from library import train_util + + # Precalculate model hashes to save time on indexing + if metadata is None: + metadata = {} + model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata) + metadata["sshs_model_hash"] = model_hash + metadata["sshs_legacy_hash"] = legacy_hash + + save_file(state_dict, file, metadata) + else: + torch.save(state_dict, file) + + def backup_weights(self): + # 重みのバックアップを行う + loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras + for lora in loras: + org_module = lora.org_module_ref[0] + if not hasattr(org_module, "_lora_org_weight"): + sd = org_module.state_dict() + org_module._lora_org_weight = sd["weight"].detach().clone() + org_module._lora_restored = True + + def restore_weights(self): + # 重みのリストアを行う + loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras + for lora in loras: + org_module = lora.org_module_ref[0] + if not org_module._lora_restored: + sd = org_module.state_dict() + sd["weight"] = org_module._lora_org_weight + org_module.load_state_dict(sd) + org_module._lora_restored = True + + def pre_calculation(self): + # 事前計算を行う + loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras + for lora in loras: + org_module = lora.org_module_ref[0] + sd = org_module.state_dict() + + org_weight = sd["weight"] + lora_weight = lora.get_weight().to(org_weight.device, dtype=org_weight.dtype) + sd["weight"] = org_weight + lora_weight + assert sd["weight"].shape == org_weight.shape + org_module.load_state_dict(sd) + + org_module._lora_restored = False + lora.enabled = False + + def apply_max_norm_regularization(self, max_norm_value, device): + downkeys = [] + upkeys = [] + alphakeys = [] + norms = [] + keys_scaled = 0 + + state_dict = self.state_dict() + for key in state_dict.keys(): + if "lora_down" in key and "weight" in key: + downkeys.append(key) + upkeys.append(key.replace("lora_down", "lora_up")) + alphakeys.append(key.replace("lora_down.weight", "alpha")) + + for i in range(len(downkeys)): + down = state_dict[downkeys[i]].to(device) + up = state_dict[upkeys[i]].to(device) + alpha = state_dict[alphakeys[i]].to(device) + dim = down.shape[0] + scale = alpha / dim + + if up.shape[2:] == (1, 1) and down.shape[2:] == (1, 1): + updown = (up.squeeze(2).squeeze(2) @ down.squeeze(2).squeeze(2)).unsqueeze(2).unsqueeze(3) + elif up.shape[2:] == (3, 3) or down.shape[2:] == (3, 3): + updown = torch.nn.functional.conv2d(down.permute(1, 0, 2, 3), up).permute(1, 0, 2, 3) + else: + updown = up @ down + + updown *= scale + + norm = updown.norm().clamp(min=max_norm_value / 2) + desired = torch.clamp(norm, max=max_norm_value) + ratio = desired.cpu() / norm.cpu() + sqrt_ratio = ratio**0.5 + if ratio != 1: + keys_scaled += 1 + state_dict[upkeys[i]] *= sqrt_ratio + state_dict[downkeys[i]] *= sqrt_ratio + scalednorm = updown.norm() * ratio + norms.append(scalednorm.item()) + + return keys_scaled, sum(norms) / len(norms), max(norms) \ No newline at end of file diff --git a/library/utils.py b/library/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3cd1f2a49cb4999a2c5bbcef9df4404128ec68f --- /dev/null +++ b/library/utils.py @@ -0,0 +1,26 @@ +import logging +import sys + + +def setup_logging(args=None, log_level=None, reset=False): + if logging.root.handlers: + if reset: + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + else: + return + + if log_level is None and args is not None: + log_level = getattr(args, "console_log_level", None) + if log_level is None: + log_level = "INFO" + log_level = getattr(logging, str(log_level).upper()) + + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(message)s")) + + logging.root.setLevel(log_level) + logging.root.addHandler(handler) + + +setup_logging() diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/attention.py b/modules/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..4595fa4ce774b95c0a8a551ced340283b4c1cccb --- /dev/null +++ b/modules/attention.py @@ -0,0 +1,133 @@ +import math + +import torch +import torch.nn.functional as F + + +try: + import flash_attn + from flash_attn.flash_attn_interface import ( + _flash_attn_forward, + flash_attn_func, + flash_attn_varlen_func, + ) +except ImportError: + flash_attn = None + flash_attn_varlen_func = None + _flash_attn_forward = None + flash_attn_func = None + +MEMORY_LAYOUT = { + # flash模式: + # 预处理: 输入 [batch_size, seq_len, num_heads, head_dim] + # 后处理: 保持形状不变 + "flash": ( + lambda x: x, # 保持形状 + lambda x: x, # 保持形状 + ), + # torch/vanilla模式: + # 预处理: 交换序列和注意力头的维度 [B,S,A,D] -> [B,A,S,D] + # 后处理: 交换回原始维度 [B,A,S,D] -> [B,S,A,D] + "torch": ( + lambda x: x.transpose(1, 2), # (B,S,A,D) -> (B,A,S,D) + lambda x: x.transpose(1, 2), # (B,A,S,D) -> (B,S,A,D) + ), + "vanilla": ( + lambda x: x.transpose(1, 2), + lambda x: x.transpose(1, 2), + ), +} + + +def attention( + q, + k, + v, + mode="torch", + drop_rate=0, + attn_mask=None, + causal=False, +): + """ + 执行QKV自注意力计算 + + Args: + q (torch.Tensor): 查询张量，形状 [batch_size, seq_len, num_heads, head_dim] + k (torch.Tensor): 键张量，形状 [batch_size, seq_len_kv, num_heads, head_dim] + v (torch.Tensor): 值张量，形状 [batch_size, seq_len_kv, num_heads, head_dim] + mode (str): 注意力模式，可选 'flash', 'torch', 'vanilla' + drop_rate (float): 注意力矩阵的dropout概率 + attn_mask (torch.Tensor): 注意力掩码，形状根据模式不同而变化 + causal (bool): 是否使用因果注意力（仅关注前面位置） + + Returns: + torch.Tensor: 注意力输出，形状 [batch_size, seq_len, num_heads * head_dim] + """ + # 获取预处理和后处理函数 + pre_attn_layout, post_attn_layout = MEMORY_LAYOUT[mode] + + # 应用预处理变换 + q = pre_attn_layout(q) # 形状根据模式变化 + k = pre_attn_layout(k) + v = pre_attn_layout(v) + + if mode == "torch": + # 使用PyTorch原生的scaled_dot_product_attention + if attn_mask is not None and attn_mask.dtype != torch.bool: + attn_mask = attn_mask.to(q.dtype) + x = F.scaled_dot_product_attention( + q, k, v, attn_mask=attn_mask, dropout_p=drop_rate, is_causal=causal + ) + elif mode == "flash": + assert flash_attn_func is not None, "flash_attn_func未定义" + assert attn_mask is None, "不支持的注意力掩码" + x: torch.Tensor = flash_attn_func( + q, k, v, dropout_p=drop_rate, causal=causal, softmax_scale=None + ) # type: ignore + elif mode == "vanilla": + # 手动实现注意力机制 + scale_factor = 1 / math.sqrt(q.size(-1)) # 缩放因子 1/sqrt(d_k) + + b, a, s, _ = q.shape # 获取形状参数 + s1 = k.size(2) # 键值序列长度 + + # 初始化注意力偏置 + attn_bias = torch.zeros(b, a, s, s1, dtype=q.dtype, device=q.device) + + # 处理因果掩码 + if causal: + assert attn_mask is None, "因果掩码和注意力掩码不能同时使用" + # 生成下三角因果掩码 + temp_mask = torch.ones(b, a, s, s, dtype=torch.bool, device=q.device).tril( + diagonal=0 + ) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + attn_bias = attn_bias.to(q.dtype) + + # 处理自定义注意力掩码 + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) + else: + attn_bias += attn_mask # 允许类似ALiBi的位置偏置 + + # 计算注意力矩阵 + attn = (q @ k.transpose(-2, -1)) * scale_factor # [B,A,S,S1] + attn += attn_bias + + # softmax和dropout + attn = attn.softmax(dim=-1) + attn = torch.dropout(attn, p=drop_rate, train=True) + + # 计算输出 + x = attn @ v # [B,A,S,D] + else: + raise NotImplementedError(f"不支持的注意力模式: {mode}") + + # 应用后处理变换 + x = post_attn_layout(x) # 恢复原始维度顺序 + + # 合并注意力头维度 + b, s, a, d = x.shape + out = x.reshape(b, s, -1) # [B,S,A*D] + return out diff --git a/modules/autoencoder.py b/modules/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..d82bcfbe14e46da8040c72c26922dfdae0d32e0c --- /dev/null +++ b/modules/autoencoder.py @@ -0,0 +1,334 @@ +# Modified from Flux +# +# Copyright 2024 Black Forest Labs + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +import torch +from einops import rearrange +from torch import Tensor, nn + + +def swish(x: Tensor) -> Tensor: + return x * torch.sigmoid(x) + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + self.in_channels = in_channels + + self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + + self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1) + + def attention(self, h_: Tensor) -> Tensor: + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + b, c, h, w = q.shape + q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous() + k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous() + v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous() + h_ = nn.functional.scaled_dot_product_attention(q, k, v) + + return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b) + + def forward(self, x: Tensor) -> Tensor: + return x + self.proj_out(self.attention(x)) + + +class ResnetBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h = x + h = self.norm1(h) + h = swish(h) + h = self.conv1(h) + + h = self.norm2(h) + h = swish(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + + return x + h + + +class Downsample(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + # no asymmetric padding in torch conv, must do it ourselves + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x: Tensor): + pad = (0, 1, 0, 1) + x = nn.functional.pad(x, pad, mode="constant", value=0) + x = self.conv(x) + return x + + +class Upsample(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor): + x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + x = self.conv(x) + return x + + +class Encoder(nn.Module): + def __init__( + self, + resolution: int, + in_channels: int, + ch: int, + ch_mult: list[int], + num_res_blocks: int, + z_channels: int, + ): + super().__init__() + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + # downsampling + self.conv_in = nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, *tuple(ch_mult)) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + block_in = self.ch + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor) -> Tensor: + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1]) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + # end + h = self.norm_out(h) + h = swish(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + def __init__( + self, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + in_channels: int, + resolution: int, + z_channels: int, + ): + super().__init__() + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.ffactor = 2 ** (self.num_resolutions - 1) + + # compute in_ch_mult, block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + + # z to block_in + self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z: Tensor) -> Tensor: + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + h = self.norm_out(h) + h = swish(h) + h = self.conv_out(h) + return h + + +class DiagonalGaussian(nn.Module): + def __init__(self, sample: bool = True, chunk_dim: int = 1): + super().__init__() + self.sample = sample + self.chunk_dim = chunk_dim + + def forward(self, z: Tensor) -> Tensor: + mean, logvar = torch.chunk(z, 2, dim=self.chunk_dim) + if self.sample: + std = torch.exp(0.5 * logvar) + return mean + std * torch.randn_like(mean) + else: + return mean + + +class AutoEncoder(nn.Module): + def __init__( + self, + resolution: int, + in_channels: int, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + z_channels: int, + scale_factor: float, + shift_factor: float, + ): + super().__init__() + self.encoder = Encoder( + resolution=resolution, + in_channels=in_channels, + ch=ch, + ch_mult=ch_mult, + num_res_blocks=num_res_blocks, + z_channels=z_channels, + ) + self.decoder = Decoder( + resolution=resolution, + in_channels=in_channels, + ch=ch, + out_ch=out_ch, + ch_mult=ch_mult, + num_res_blocks=num_res_blocks, + z_channels=z_channels, + ) + self.reg = DiagonalGaussian() + + self.scale_factor = scale_factor + self.shift_factor = shift_factor + + @property + def device(self) -> torch.device: + return next(self.parameters()).device + + @property + def dtype(self) -> torch.dtype: + return next(self.parameters()).dtype + + def encode(self, x: Tensor) -> Tensor: + z = self.reg(self.encoder(x)) + z = self.scale_factor * (z - self.shift_factor) + return z + + def decode(self, z: Tensor) -> Tensor: + z = z / self.scale_factor + self.shift_factor + return self.decoder(z) + + def forward(self, x: Tensor) -> Tensor: + return self.decode(self.encode(x)) diff --git a/modules/conditioner.py b/modules/conditioner.py new file mode 100644 index 0000000000000000000000000000000000000000..3532a02e8ac19856e20e3857705cb883b865bc74 --- /dev/null +++ b/modules/conditioner.py @@ -0,0 +1,239 @@ +import torch +from qwen_vl_utils import process_vision_info +from transformers import ( + AutoProcessor, + Qwen2VLForConditionalGeneration, + Qwen2_5_VLForConditionalGeneration, +) +from torchvision.transforms import ToPILImage + +to_pil = ToPILImage() + +Qwen25VL_7b_PREFIX = '''You are an expert image analyst specializing in 3D scene understanding. Your goal is to describe the 3D structure and layout of the scene in the image to aid in depth estimation tasks. +- Focus your description on elements crucial for 3D understanding. Incorporate observations about key **objects**, their **spatial relationships** (like positions, relative distances, and relative sizes using real-world orientation), the overall **layout**, potential **camera perspective**, and visible **depth cues** (like occlusion or perspective lines) into a unified description. +- Be concise but informative, and assume it is a real-world image. Only describe what you are very confident about. +User Prompt:''' + +Qwen25VL_7b_PREFIX2 = '''Given a user prompt, generate an "Enhanced prompt" that provides detailed visual descriptions suitable for image generation. Evaluate the level of detail in the user prompt: +- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, and spatial relationships to create vivid and concrete scenes. +- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating.\n +Here are examples of how to transform or refine prompts: +- User Prompt: A cat sleeping -> Enhanced: A small, fluffy white cat curled up in a round shape, sleeping peacefully on a warm sunny windowsill, surrounded by pots of blooming red flowers. +- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street lamps, a diverse crowd of people in colorful clothing, and a double-decker bus passing by towering glass skyscrapers.\n +Please generate only the enhanced description for the prompt below and avoid including any additional commentary or evaluations: +User Prompt:''' + +Qwen25VL_7b_PREFIX3 = '''You are a helpful assistant. Describe the image in detail. User Prompt:''' + +def split_string(s): + # 将中文引号替换为英文引号 + s = s.replace("“", '"').replace("”", '"') # use english quotes + result = [] + # 标记是否在引号内 + in_quotes = False + temp = "" + + # 遍历字符串中的每个字符及其索引 + for idx, char in enumerate(s): + # 如果字符是引号且索引大于 155 + if char == '"' and idx > 155: + # 将引号添加到临时字符串 + temp += char + # 如果不在引号内 + if not in_quotes: + # 将临时字符串添加到结果列表 + result.append(temp) + # 清空临时字符串 + temp = "" + + # 切换引号状态 + in_quotes = not in_quotes + continue + # 如果在引号内 + if in_quotes: + # 如果字符是空格 + if char.isspace(): + pass # have space token + + # 将字符用中文引号包裹后添加到结果列表 + result.append("“" + char + "”") + else: + # 将字符添加到临时字符串 + temp += char + + # 如果临时字符串不为空 + if temp: + # 将临时字符串添加到结果列表 + result.append(temp) + + return result + + +class Qwen25VL_7b_Embedder(torch.nn.Module): + def __init__(self, model_path, max_length=640, dtype=torch.bfloat16, device="cuda",args=None): + super(Qwen25VL_7b_Embedder, self).__init__() + self.max_length = max_length + + self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_path, + torch_dtype=dtype, + attn_implementation="flash_attention_2", + ).to(torch.cuda.current_device()) + + self.model.requires_grad_(False) + self.processor = AutoProcessor.from_pretrained( + model_path, min_pixels=256 * 28 * 28, max_pixels=324 * 28 * 28 + ) + + self.prefix = Qwen25VL_7b_PREFIX if (args is not None) and (not args.old_prompt) else Qwen25VL_7b_PREFIX2 + self.prefix = Qwen25VL_7b_PREFIX3 if (args is not None) and (args.prompt_type == "rgb" or args.prompt_type == "empty") else self.prefix + self.prefix_len = self.calculate_len_of_prefix() + + @property + def device(self) -> torch.device: + return next(self.parameters()).device + + @property + def dtype(self) -> torch.dtype: + return next(self.parameters()).dtype + + def calculate_len_of_prefix(self): + messages_specific = [{"role": "user", "content": []}] + messages_specific[0]["content"].append({"type": "text", "text": f"{self.prefix}"}) + + text_specific = self.processor.apply_chat_template( + messages_specific, tokenize=False, add_generation_prompt=True, add_vision_id=True + ) + + + inputs_specific = self.processor( + text=[text_specific], + padding=True, + # return_tensors="pt", + ) + len_specific = len(inputs_specific["input_ids"][0]) - 6 # 223 - 6 = self.prefix_len + return len_specific + + def forward(self, caption, ref_images): + text_list = caption + embs = torch.zeros( + len(text_list), + self.max_length, + self.model.config.hidden_size, + dtype=torch.bfloat16, + device=torch.cuda.current_device(), + ) + hidden_states = torch.zeros( + len(text_list), + self.max_length, + self.model.config.hidden_size, + dtype=torch.bfloat16, + device=torch.cuda.current_device(), + ) + masks = torch.zeros( + len(text_list), + self.max_length, + dtype=torch.long, + device=torch.cuda.current_device(), + ) + input_ids_list = [] + attention_mask_list = [] + emb_list = [] + + def split_string(s): + s = s.replace("“", '"').replace("”", '"').replace("'", '''"''') # use english quotes + result = [] + in_quotes = False + temp = "" + + for idx,char in enumerate(s): + if char == '"' and idx>155: + temp += char + if not in_quotes: + result.append(temp) + temp = "" + + in_quotes = not in_quotes + continue + if in_quotes: + if char.isspace(): + pass # have space token + + result.append("“" + char + "”") + else: + temp += char + + if temp: + result.append(temp) + + return result + + + for idx, (txt, imgs) in enumerate(zip(text_list, ref_images)): + + messages = [{"role": "user", "content": []}] + messages[0]["content"].append({"type": "text", "text": f"{self.prefix}"}) + messages[0]["content"].append({"type": "image", "image": to_pil(imgs)}) + # 再添加 text + messages[0]["content"].append({"type": "text", "text": f"{txt}"}) + # Preparation for inference + text = self.processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, add_vision_id=True + ) + image_inputs, video_inputs = process_vision_info(messages) + inputs = self.processor( + text=[text], + images=image_inputs, + padding=True, + return_tensors="pt", + ) + + old_inputs_ids = inputs.input_ids + text_split_list = split_string(text) + token_list = [] + for text_each in text_split_list: + txt_inputs = self.processor( + text=text_each, + images=None, + videos=None, + padding=True, + return_tensors="pt", + ) + token_each = txt_inputs.input_ids + if token_each[0][0] == 2073 and token_each[0][-1] == 854: + token_each = token_each[:, 1:-1] + token_list.append(token_each) + else: + token_list.append(token_each) + + new_txt_ids = torch.cat(token_list, dim=1).to("cuda") + + new_txt_ids = new_txt_ids.to(old_inputs_ids.device) + + idx1 = (old_inputs_ids == 151653).nonzero(as_tuple=True)[1][0] + idx2 = (new_txt_ids == 151653).nonzero(as_tuple=True)[1][0] + inputs.input_ids = ( + torch.cat([old_inputs_ids[0, :idx1], new_txt_ids[0, idx2:]], dim=0) + .unsqueeze(0) + .to("cuda") + ) + inputs.attention_mask = (inputs.input_ids > 0).long().to("cuda") + outputs = self.model( + input_ids=inputs.input_ids, + attention_mask=inputs.attention_mask, + pixel_values=inputs.pixel_values.to("cuda"), + image_grid_thw=inputs.image_grid_thw.to("cuda"), + output_hidden_states=True, + ) + + emb = outputs["hidden_states"][-1] + + embs[idx, : min(self.max_length, emb.shape[1] - self.prefix_len)] = emb[0, self.prefix_len:][:self.max_length] #2,640,3584 + + masks[idx, : min(self.max_length, emb.shape[1] - self.prefix_len)] = torch.ones( + (min(self.max_length, emb.shape[1] - self.prefix_len)), + dtype=torch.long, + device=torch.cuda.current_device(), + ) + + return embs, masks \ No newline at end of file diff --git a/modules/connector_edit.py b/modules/connector_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..7aa589d7ccba1cc2517f25537076e97362df98e4 --- /dev/null +++ b/modules/connector_edit.py @@ -0,0 +1,486 @@ +from typing import Optional + +import torch +import torch.nn +from einops import rearrange +from torch import nn + +from .layers import MLP, TextProjection, TimestepEmbedder, apply_gate, attention + + +class RMSNorm(nn.Module): + def __init__( + self, + dim: int, + elementwise_affine=True, + eps: float = 1e-6, + device=None, + dtype=None, + ): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs)) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +def get_norm_layer(norm_layer): + """ + Get the normalization layer. + + Args: + norm_layer (str): The type of normalization layer. + + Returns: + norm_layer (nn.Module): The normalization layer. + """ + if norm_layer == "layer": + return nn.LayerNorm + elif norm_layer == "rms": + return RMSNorm + else: + raise NotImplementedError(f"Norm layer {norm_layer} is not implemented") + + +def get_activation_layer(act_type): + """get activation layer + + Args: + act_type (str): the activation type + + Returns: + torch.nn.functional: the activation layer + """ + if act_type == "gelu": + return lambda: nn.GELU() + elif act_type == "gelu_tanh": + return lambda: nn.GELU(approximate="tanh") + elif act_type == "relu": + return nn.ReLU + elif act_type == "silu": + return nn.SiLU + else: + raise ValueError(f"Unknown activation type: {act_type}") + +class IndividualTokenRefinerBlock(torch.nn.Module): + def __init__( + self, + hidden_size, + heads_num, + mlp_width_ratio: str = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + need_CA: bool = False, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.need_CA = need_CA + self.heads_num = heads_num + head_dim = hidden_size // heads_num + mlp_hidden_dim = int(hidden_size * mlp_width_ratio) + + self.norm1 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + self.self_attn_qkv = nn.Linear( + hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs + ) + qk_norm_layer = get_norm_layer(qk_norm_type) + self.self_attn_q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_proj = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + self.norm2 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + act_layer = get_activation_layer(act_type) + self.mlp = MLP( + in_channels=hidden_size, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=mlp_drop_rate, + **factory_kwargs, + ) + + self.adaLN_modulation = nn.Sequential( + act_layer(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True, **factory_kwargs), + ) + + if self.need_CA: + self.cross_attnblock=CrossAttnBlock(hidden_size=hidden_size, + heads_num=heads_num, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + act_type=act_type, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + **factory_kwargs,) + # Zero-initialize the modulation + nn.init.zeros_(self.adaLN_modulation[1].weight) + nn.init.zeros_(self.adaLN_modulation[1].bias) + + def forward( + self, + x: torch.Tensor, + c: torch.Tensor, # timestep_aware_representations + context_aware_representations + attn_mask: torch.Tensor = None, + y: torch.Tensor = None, + ): + gate_msa, gate_mlp = self.adaLN_modulation(c).chunk(2, dim=1) + + norm_x = self.norm1(x) + qkv = self.self_attn_qkv(norm_x) + q, k, v = rearrange(qkv, "B L (K H D) -> K B L H D", K=3, H=self.heads_num) + # Apply QK-Norm if needed + q = self.self_attn_q_norm(q).to(v) + k = self.self_attn_k_norm(k).to(v) + + # Self-Attention + attn = attention(q, k, v, mode="torch", attn_mask=attn_mask) + + x = x + apply_gate(self.self_attn_proj(attn), gate_msa) + + if self.need_CA: + x = self.cross_attnblock(x, c, attn_mask, y) + + # FFN Layer + x = x + apply_gate(self.mlp(self.norm2(x)), gate_mlp) + + return x + + + + +class CrossAttnBlock(torch.nn.Module): + def __init__( + self, + hidden_size, + heads_num, + mlp_width_ratio: str = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.heads_num = heads_num + head_dim = hidden_size // heads_num + + self.norm1 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + self.norm1_2 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + self.self_attn_q = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + self.self_attn_kv = nn.Linear( + hidden_size, hidden_size*2, bias=qkv_bias, **factory_kwargs + ) + qk_norm_layer = get_norm_layer(qk_norm_type) + self.self_attn_q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) + if qk_norm + else nn.Identity() + ) + self.self_attn_proj = nn.Linear( + hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs + ) + + self.norm2 = nn.LayerNorm( + hidden_size, elementwise_affine=True, eps=1e-6, **factory_kwargs + ) + act_layer = get_activation_layer(act_type) + + self.adaLN_modulation = nn.Sequential( + act_layer(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True, **factory_kwargs), + ) + # Zero-initialize the modulation + nn.init.zeros_(self.adaLN_modulation[1].weight) + nn.init.zeros_(self.adaLN_modulation[1].bias) + + def forward( + self, + x: torch.Tensor, + c: torch.Tensor, # timestep_aware_representations + context_aware_representations + attn_mask: torch.Tensor = None, + y: torch.Tensor=None, + + ): + gate_msa, gate_mlp = self.adaLN_modulation(c).chunk(2, dim=1) + + norm_x = self.norm1(x) + norm_y = self.norm1_2(y) + q = self.self_attn_q(norm_x) + q = rearrange(q, "B L (H D) -> B L H D", H=self.heads_num) + kv = self.self_attn_kv(norm_y) + k, v = rearrange(kv, "B L (K H D) -> K B L H D", K=2, H=self.heads_num) + # Apply QK-Norm if needed + q = self.self_attn_q_norm(q).to(v) + k = self.self_attn_k_norm(k).to(v) + + # Self-Attention + attn = attention(q, k, v, mode="torch", attn_mask=attn_mask) + + x = x + apply_gate(self.self_attn_proj(attn), gate_msa) + + return x + + + +class IndividualTokenRefiner(torch.nn.Module): + def __init__( + self, + hidden_size, + heads_num, + depth, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + need_CA:bool=False, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.need_CA = need_CA + self.blocks = nn.ModuleList( + [ + IndividualTokenRefinerBlock( + hidden_size=hidden_size, + heads_num=heads_num, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + act_type=act_type, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + need_CA=self.need_CA, + **factory_kwargs, + ) + for _ in range(depth) + ] + ) + + + def forward( + self, + x: torch.Tensor, + c: torch.LongTensor, + mask: Optional[torch.Tensor] = None, + y:torch.Tensor=None, + ): + self_attn_mask = None + if mask is not None: + batch_size = mask.shape[0] + seq_len = mask.shape[1] + mask = mask.to(x.device) + # batch_size x 1 x seq_len x seq_len + self_attn_mask_1 = mask.view(batch_size, 1, 1, seq_len).repeat( + 1, 1, seq_len, 1 + ) + # batch_size x 1 x seq_len x seq_len + self_attn_mask_2 = self_attn_mask_1.transpose(2, 3) + # batch_size x 1 x seq_len x seq_len, 1 for broadcasting of heads_num + self_attn_mask = (self_attn_mask_1 & self_attn_mask_2).bool() + # avoids self-attention weight being NaN for padding tokens + self_attn_mask[:, :, :, 0] = True + + + for block in self.blocks: + x = block(x, c, self_attn_mask,y) + + return x + + +class SingleTokenRefiner(torch.nn.Module): + """ + A single token refiner block for llm text embedding refine. + """ + def __init__( + self, + in_channels, + hidden_size, + heads_num, + depth, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + act_type: str = "silu", + qk_norm: bool = False, + qk_norm_type: str = "layer", + qkv_bias: bool = True, + need_CA:bool=False, + attn_mode: str = "torch", + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.attn_mode = attn_mode + self.need_CA = need_CA + assert self.attn_mode == "torch", "Only support 'torch' mode for token refiner." + + self.input_embedder = nn.Linear( + in_channels, hidden_size, bias=True, **factory_kwargs + ) + if self.need_CA: + self.input_embedder_CA = nn.Linear( + in_channels, hidden_size, bias=True, **factory_kwargs + ) + + act_layer = get_activation_layer(act_type) + # Build timestep embedding layer + self.t_embedder = TimestepEmbedder(hidden_size, act_layer, **factory_kwargs) + # Build context embedding layer + self.c_embedder = TextProjection( + in_channels, hidden_size, act_layer, **factory_kwargs + ) + + self.individual_token_refiner = IndividualTokenRefiner( + hidden_size=hidden_size, + heads_num=heads_num, + depth=depth, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + act_type=act_type, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + qkv_bias=qkv_bias, + need_CA=need_CA, + **factory_kwargs, + ) + + def forward( + self, + x: torch.Tensor, + t: torch.LongTensor, + mask: Optional[torch.LongTensor] = None, + y: torch.LongTensor=None, + ): + timestep_aware_representations = self.t_embedder(t) + + if mask is None: + context_aware_representations = x.mean(dim=1) + else: + mask_float = mask.unsqueeze(-1) # [b, s1, 1] + context_aware_representations = (x * mask_float).sum( + dim=1 + ) / mask_float.sum(dim=1) + context_aware_representations = self.c_embedder(context_aware_representations) + c = timestep_aware_representations + context_aware_representations + + x = self.input_embedder(x) + if self.need_CA: + y = self.input_embedder_CA(y) + x = self.individual_token_refiner(x, c, mask, y) + else: + x = self.individual_token_refiner(x, c, mask) + + return x + + + +class Qwen2Connector(torch.nn.Module): + def __init__( + self, + in_channels=3584, + hidden_size=4096, + heads_num=32, + depth=2, + need_CA=False, + device=None, + dtype=torch.bfloat16, + ): + super().__init__() + factory_kwargs = {"device": device, "dtype":dtype} + + self.S =SingleTokenRefiner(in_channels=in_channels,hidden_size=hidden_size,heads_num=heads_num,depth=depth,need_CA=need_CA,**factory_kwargs) + self.global_proj_out=nn.Linear(in_channels,768) + + self.scale_factor = nn.Parameter(torch.zeros(1)) + with torch.no_grad(): + self.scale_factor.data += -(1 - 0.09) + + def forward(self, x,t,mask): + t = t * 1000 # fix the times embedding bug + mask_float = mask.unsqueeze(-1) # [b, s1, 1] + x_mean = (x * mask_float).sum( + dim=1 + ) / mask_float.sum(dim=1) * (1 + self.scale_factor.to(x.dtype)) + + global_out=self.global_proj_out(x_mean) + encoder_hidden_states = self.S(x,t,mask) + return encoder_hidden_states,global_out \ No newline at end of file diff --git a/modules/layers.py b/modules/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..47c78ce49c7264845c866150a580f08dbbba715d --- /dev/null +++ b/modules/layers.py @@ -0,0 +1,742 @@ +# Modified from Flux +# +# Copyright 2024 Black Forest Labs + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import math # noqa: I001 +from dataclasses import dataclass +from functools import partial + +import torch +import torch.nn.functional as F +from einops import rearrange +try: + from liger_kernel.ops.rms_norm import LigerRMSNormFunction +except ImportError: + LigerRMSNormFunction = None +from torch import Tensor, nn +from torch.utils.checkpoint import checkpoint + +try: + import flash_attn + from flash_attn.flash_attn_interface import ( + _flash_attn_forward, + flash_attn_varlen_func, + ) +except ImportError: + flash_attn = None + flash_attn_varlen_func = None + _flash_attn_forward = None + +def to_cuda(x): + if isinstance(x, torch.Tensor): + return x.cuda() + elif isinstance(x, (list, tuple)): + return [to_cuda(elem) for elem in x] + elif isinstance(x, dict): + return {k: to_cuda(v) for k, v in x.items()} + else: + return x + + +def to_cpu(x): + if isinstance(x, torch.Tensor): + return x.cpu() + elif isinstance(x, (list, tuple)): + return [to_cpu(elem) for elem in x] + elif isinstance(x, dict): + return {k: to_cpu(v) for k, v in x.items()} + else: + return x + +MEMORY_LAYOUT = { + "flash": ( + lambda x: x.view(x.shape[0] * x.shape[1], *x.shape[2:]), + lambda x: x, + ), + "torch": ( + lambda x: x.transpose(1, 2), + lambda x: x.transpose(1, 2), + ), + "vanilla": ( + lambda x: x.transpose(1, 2), + lambda x: x.transpose(1, 2), + ), +} + + +def attention( + q, + k, + v, + mode="torch", + drop_rate=0, + attn_mask=None, + causal=False, + cu_seqlens_q=None, + cu_seqlens_kv=None, + max_seqlen_q=None, + max_seqlen_kv=None, + batch_size=1, +): + """ + Perform QKV self attention. + + Args: + q (torch.Tensor): Query tensor with shape [b, s, a, d], where a is the number of heads. + k (torch.Tensor): Key tensor with shape [b, s1, a, d] + v (torch.Tensor): Value tensor with shape [b, s1, a, d] + mode (str): Attention mode. Choose from 'self_flash', 'cross_flash', 'torch', and 'vanilla'. + drop_rate (float): Dropout rate in attention map. (default: 0) + attn_mask (torch.Tensor): Attention mask with shape [b, s1] (cross_attn), or [b, a, s, s1] (torch or vanilla). + (default: None) + causal (bool): Whether to use causal attention. (default: False) + cu_seqlens_q (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, + used to index into q. + cu_seqlens_kv (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, + used to index into kv. + max_seqlen_q (int): The maximum sequence length in the batch of q. + max_seqlen_kv (int): The maximum sequence length in the batch of k and v. + + Returns: + torch.Tensor: Output tensor after self attention with shape [b, s, ad] + """ + pre_attn_layout, post_attn_layout = MEMORY_LAYOUT[mode] + q = pre_attn_layout(q) + k = pre_attn_layout(k) + v = pre_attn_layout(v) + + if mode == "torch": + if attn_mask is not None and attn_mask.dtype != torch.bool: + attn_mask = attn_mask.to(q.dtype) + x = F.scaled_dot_product_attention( + q, k, v, attn_mask=attn_mask, dropout_p=drop_rate, is_causal=causal + ) + elif mode == "flash": + assert flash_attn_varlen_func is not None + x: torch.Tensor = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + ) # type: ignore + # x with shape [(bxs), a, d] + x = x.view(batch_size, max_seqlen_q, x.shape[-2], x.shape[-1]) # type: ignore # reshape x to [b, s, a, d] + elif mode == "vanilla": + scale_factor = 1 / math.sqrt(q.size(-1)) + + b, a, s, _ = q.shape + s1 = k.size(2) + attn_bias = torch.zeros(b, a, s, s1, dtype=q.dtype, device=q.device) + if causal: + # Only applied to self attention + assert attn_mask is None, ( + "Causal mask and attn_mask cannot be used together" + ) + temp_mask = torch.ones(b, a, s, s, dtype=torch.bool, device=q.device).tril( + diagonal=0 + ) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + attn_bias.to(q.dtype) + + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) + else: + attn_bias += attn_mask + + # TODO: Maybe force q and k to be float32 to avoid numerical overflow + attn = (q @ k.transpose(-2, -1)) * scale_factor + attn += attn_bias + attn = attn.softmax(dim=-1) + attn = torch.dropout(attn, p=drop_rate, train=True) + x = attn @ v + else: + raise NotImplementedError(f"Unsupported attention mode: {mode}") + + x = post_attn_layout(x) + b, s, a, d = x.shape + out = x.reshape(b, s, -1) + return out + + +def apply_gate(x, gate=None, tanh=False): + """AI is creating summary for apply_gate + + Args: + x (torch.Tensor): input tensor. + gate (torch.Tensor, optional): gate tensor. Defaults to None. + tanh (bool, optional): whether to use tanh function. Defaults to False. + + Returns: + torch.Tensor: the output tensor after apply gate. + """ + if gate is None: + return x + if tanh: + return x * gate.unsqueeze(1).tanh() + else: + return x * gate.unsqueeze(1) + + +class MLP(nn.Module): + """MLP as used in Vision Transformer, MLP-Mixer and related networks""" + + def __init__( + self, + in_channels, + hidden_channels=None, + out_features=None, + act_layer=nn.GELU, + norm_layer=None, + bias=True, + drop=0.0, + use_conv=False, + device=None, + dtype=None, + ): + super().__init__() + out_features = out_features or in_channels + hidden_channels = hidden_channels or in_channels + bias = (bias, bias) + drop_probs = (drop, drop) + linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear + + self.fc1 = linear_layer( + in_channels, hidden_channels, bias=bias[0], device=device, dtype=dtype + ) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs[0]) + self.norm = ( + norm_layer(hidden_channels, device=device, dtype=dtype) + if norm_layer is not None + else nn.Identity() + ) + self.fc2 = linear_layer( + hidden_channels, out_features, bias=bias[1], device=device, dtype=dtype + ) + self.drop2 = nn.Dropout(drop_probs[1]) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.norm(x) + x = self.fc2(x) + x = self.drop2(x) + return x + + +class TextProjection(nn.Module): + """ + Projects text embeddings. Also handles dropout for classifier-free guidance. + + Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py + """ + + def __init__(self, in_channels, hidden_size, act_layer, dtype=None, device=None): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.linear_1 = nn.Linear( + in_features=in_channels, + out_features=hidden_size, + bias=True, + **factory_kwargs, + ) + self.act_1 = act_layer() + self.linear_2 = nn.Linear( + in_features=hidden_size, + out_features=hidden_size, + bias=True, + **factory_kwargs, + ) + + def forward(self, caption): + hidden_states = self.linear_1(caption) + hidden_states = self.act_1(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__( + self, + hidden_size, + act_layer, + frequency_embedding_size=256, + max_period=10000, + out_size=None, + dtype=None, + device=None, + ): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.frequency_embedding_size = frequency_embedding_size + self.max_period = max_period + if out_size is None: + out_size = hidden_size + + self.mlp = nn.Sequential( + nn.Linear( + frequency_embedding_size, hidden_size, bias=True, **factory_kwargs + ), + act_layer(), + nn.Linear(hidden_size, out_size, bias=True, **factory_kwargs), + ) + nn.init.normal_(self.mlp[0].weight, std=0.02) # type: ignore + nn.init.normal_(self.mlp[2].weight, std=0.02) # type: ignore + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + + Args: + t (torch.Tensor): a 1-D Tensor of N indices, one per batch element. These may be fractional. + dim (int): the dimension of the output. + max_period (int): controls the minimum frequency of the embeddings. + + Returns: + embedding (torch.Tensor): An (N, D) Tensor of positional embeddings. + + .. ref_link: https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half + ).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1 + ) + return embedding + + def forward(self, t): + t_freq = self.timestep_embedding( + t, self.frequency_embedding_size, self.max_period + ).type(self.mlp[0].weight.dtype) # type: ignore + t_emb = self.mlp(t_freq) + return t_emb + + +class EmbedND(nn.Module): + def __init__(self, dim: int, theta: int, axes_dim: list[int]): + super().__init__() + self.dim = dim + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: Tensor) -> Tensor: + n_axes = ids.shape[-1] + emb = torch.cat( + [rope(ids[..., i], self.axes_dim[i], self.theta) for i in range(n_axes)], + dim=-3, + ) + + return emb.unsqueeze(1) + + +class MLPEmbedder(nn.Module): + def __init__(self, in_dim: int, hidden_dim: int): + super().__init__() + self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True) + self.silu = nn.SiLU() + self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True) + + self.gradient_checkpointing = False + + def enable_gradient_checkpointing(self): + self.gradient_checkpointing = True + + def disable_gradient_checkpointing(self): + self.gradient_checkpointing = False + + def _forward(self, x: Tensor) -> Tensor: + return self.out_layer(self.silu(self.in_layer(x))) + + def forward(self, *args, **kwargs): + if self.training and self.gradient_checkpointing: + return checkpoint(self._forward, *args, use_reentrant=False, **kwargs) + else: + return self._forward(*args, **kwargs) + + +def rope(pos, dim: int, theta: int): + assert dim % 2 == 0 + scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim + omega = 1.0 / (theta**scale) + out = torch.einsum("...n,d->...nd", pos, omega) + out = torch.stack( + [torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1 + ) + out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) + return out.float() + + +def attention_after_rope(q, k, v, pe): + q, k = apply_rope(q, k, pe) + + from .attention import attention + + x = attention(q, k, v, mode="torch") + return x + + +# @torch.compile(mode="max-autotune-no-cudagraphs", dynamic=True) +def apply_rope(xq, xk, freqs_cis): + # 将 num_heads 和 seq_len 的维度交换回原函数的处理顺序 + xq = xq.transpose(1, 2) # [batch, num_heads, seq_len, head_dim] + xk = xk.transpose(1, 2) + + # 将 head_dim 拆分为复数部分（实部和虚部） + xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2) + xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2) + + # 应用旋转位置编码（复数乘法） + xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] + xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1] + + # 恢复张量形状并转置回目标维度顺序 + xq_out = xq_out.reshape(*xq.shape).type_as(xq).transpose(1, 2) + xk_out = xk_out.reshape(*xk.shape).type_as(xk).transpose(1, 2) + + return xq_out, xk_out + + +# @torch.compile(mode="max-autotune-no-cudagraphs", dynamic=True) +def scale_add_residual( + x: torch.Tensor, scale: torch.Tensor, residual: torch.Tensor +) -> torch.Tensor: + return x * scale + residual + + +# @torch.compile(mode="max-autotune-no-cudagraphs", dynamic=True) +def layernorm_and_scale_shift( + x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor +) -> torch.Tensor: + return torch.nn.functional.layer_norm(x, (x.size(-1),)) * (scale + 1) + shift + + +class SelfAttention(nn.Module): + def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.norm = QKNorm(head_dim) + self.proj = nn.Linear(dim, dim) + + def forward(self, x: Tensor, pe: Tensor) -> Tensor: + qkv = self.qkv(x) + q, k, v = rearrange(qkv, "B L (K H D) -> K B L H D", K=3, H=self.num_heads) + q, k = self.norm(q, k, v) + x = attention_after_rope(q, k, v, pe=pe) + x = self.proj(x) + return x + + +@dataclass +class ModulationOut: + shift: Tensor + scale: Tensor + gate: Tensor + + +class RMSNorm(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.scale = nn.Parameter(torch.ones(dim)) + + @staticmethod + def rms_norm_fast(x, weight, eps): + return LigerRMSNormFunction.apply( + x, + weight, + eps, + 0.0, + "gemma", + True, + ) + + @staticmethod + def rms_norm(x, weight, eps): + x_dtype = x.dtype + x = x.float() + rrms = torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + eps) + return (x * rrms).to(dtype=x_dtype) * weight + + def forward(self, x: Tensor): + if LigerRMSNormFunction is not None: + return self.rms_norm_fast(x, self.scale.to(x.dtype), 1e-6) + return self.rms_norm(x, self.scale.to(x.dtype), 1e-6) + + +class QKNorm(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.query_norm = RMSNorm(dim) + self.key_norm = RMSNorm(dim) + + def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple[Tensor, Tensor]: + q = self.query_norm(q) + k = self.key_norm(k) + return q.to(v), k.to(v) + + +class Modulation(nn.Module): + def __init__(self, dim: int, double: bool): + super().__init__() + self.is_double = double + self.multiplier = 6 if double else 3 + self.lin = nn.Linear(dim, self.multiplier * dim, bias=True) + + def forward(self, vec: Tensor) -> tuple[ModulationOut, ModulationOut | None]: + out = self.lin(nn.functional.silu(vec))[:, None, :].chunk( + self.multiplier, dim=-1 + ) + + return ( + ModulationOut(*out[:3]), + ModulationOut(*out[3:]) if self.is_double else None, + ) + + +class DoubleStreamBlock(nn.Module): + def __init__( + self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False + ): + super().__init__() + + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.num_heads = num_heads + self.hidden_size = hidden_size + self.img_mod = Modulation(hidden_size, double=True) + self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.img_attn = SelfAttention( + dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias + ) + + self.img_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.img_mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.GELU(approximate="tanh"), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + ) + + self.txt_mod = Modulation(hidden_size, double=True) + self.txt_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.txt_attn = SelfAttention( + dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias + ) + + self.txt_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.txt_mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.GELU(approximate="tanh"), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + ) + + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + + def enable_gradient_checkpointing(self, cpu_offload: bool = False): + self.gradient_checkpointing = True + self.cpu_offload_checkpointing = cpu_offload + + def disable_gradient_checkpointing(self): + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + + def _forward( + self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor + ) -> tuple[Tensor, Tensor]: + img_mod1, img_mod2 = self.img_mod(vec) + txt_mod1, txt_mod2 = self.txt_mod(vec) + + # prepare image for attention + img_modulated = self.img_norm1(img) + img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift + img_qkv = self.img_attn.qkv(img_modulated) + img_q, img_k, img_v = rearrange( + img_qkv, "B L (K H D) -> K B L H D", K=3, H=self.num_heads + ) + img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) + + # prepare txt for attention + txt_modulated = self.txt_norm1(txt) + txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift + txt_qkv = self.txt_attn.qkv(txt_modulated) + txt_q, txt_k, txt_v = rearrange( + txt_qkv, "B L (K H D) -> K B L H D", K=3, H=self.num_heads + ) + txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) + + # run actual attention + q = torch.cat((txt_q, img_q), dim=1) + k = torch.cat((txt_k, img_k), dim=1) + v = torch.cat((txt_v, img_v), dim=1) + + attn = attention_after_rope(q, k, v, pe=pe) + txt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1] :] + + # calculate the img bloks + img = img + img_mod1.gate * self.img_attn.proj(img_attn) + img_mlp = self.img_mlp( + (1 + img_mod2.scale) * self.img_norm2(img) + img_mod2.shift + ) + img = scale_add_residual(img_mlp, img_mod2.gate, img) + + # calculate the txt bloks + txt = txt + txt_mod1.gate * self.txt_attn.proj(txt_attn) + txt_mlp = self.txt_mlp( + (1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift + ) + txt = scale_add_residual(txt_mlp, txt_mod2.gate, txt) + return img, txt + + def forward( + self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor + ) -> tuple[Tensor, Tensor]: + if self.training and self.gradient_checkpointing: + if not self.cpu_offload_checkpointing: + return checkpoint(self._forward, img, txt, vec, pe, use_reentrant=False) + # cpu offload checkpointing + + def create_custom_forward(func): + def custom_forward(*inputs): + cuda_inputs = to_cuda(inputs) + outputs = func(*cuda_inputs) + return to_cpu(outputs) + + return custom_forward + + return torch.utils.checkpoint.checkpoint( + create_custom_forward(self._forward), img, txt, vec, pe, use_reentrant=False + ) + + else: + return self._forward(img, txt, vec, pe) + +class SingleStreamBlock(nn.Module): + """ + A DiT block with parallel linear layers as described in + https://arxiv.org/abs/2302.05442 and adapted modulation interface. + """ + + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 4.0, + qk_scale: float | None = None, + ): + super().__init__() + self.hidden_dim = hidden_size + self.num_heads = num_heads + head_dim = hidden_size // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.mlp_hidden_dim = int(hidden_size * mlp_ratio) + # qkv and mlp_in + self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim) + # proj and mlp_out + self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size) + + self.norm = QKNorm(head_dim) + + self.hidden_size = hidden_size + self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + + self.mlp_act = nn.GELU(approximate="tanh") + self.modulation = Modulation(hidden_size, double=False) + + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + + def enable_gradient_checkpointing(self, cpu_offload: bool = False): + self.gradient_checkpointing = True + self.cpu_offload_checkpointing = cpu_offload + + def disable_gradient_checkpointing(self): + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + + def _forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: + mod, _ = self.modulation(vec) + x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift + qkv, mlp = torch.split( + self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1 + ) + + q, k, v = rearrange(qkv, "B L (K H D) -> K B L H D", K=3, H=self.num_heads) + q, k = self.norm(q, k, v) + + # compute attention + attn = attention_after_rope(q, k, v, pe=pe) + # compute activation in mlp stream, cat again and run second linear layer + output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) + return scale_add_residual(output, mod.gate, x) + + def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: + if self.training and self.gradient_checkpointing: + if not self.cpu_offload_checkpointing: + return checkpoint(self._forward, x, vec, pe, use_reentrant=False) + + # cpu offload checkpointing + + def create_custom_forward(func): + def custom_forward(*inputs): + cuda_inputs = to_cuda(inputs) + outputs = func(*cuda_inputs) + return to_cpu(outputs) + + return custom_forward + + return torch.utils.checkpoint.checkpoint( + create_custom_forward(self._forward), x, vec, pe, use_reentrant=False + ) + else: + return self._forward(x, vec, pe) + +class LastLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear( + hidden_size, patch_size * patch_size * out_channels, bias=True + ) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True) + ) + + def forward(self, x: Tensor, vec: Tensor) -> Tensor: + shift, scale = self.adaLN_modulation(vec).chunk(2, dim=1) + x = (1 + scale[:, None, :]) * self.norm_final(x) + shift[:, None, :] + x = self.linear(x) + return x diff --git a/modules/model_edit.py b/modules/model_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..3d0b32f04ccfe84188cc3304c552a414d4851468 --- /dev/null +++ b/modules/model_edit.py @@ -0,0 +1,229 @@ +import math +from dataclasses import dataclass + +import numpy as np +import torch + +from library import custom_offloading_utils + +from torch import Tensor, nn + +from .connector_edit import Qwen2Connector +from .layers import DoubleStreamBlock, EmbedND, LastLayer, MLPEmbedder, SingleStreamBlock + + +@dataclass +class Step1XParams: + in_channels: int + out_channels: int + vec_in_dim: int + context_in_dim: int + hidden_size: int + mlp_ratio: float + num_heads: int + depth: int + depth_single_blocks: int + axes_dim: list[int] + theta: int + qkv_bias: bool + + +class Step1XEdit(nn.Module): + """ + Transformer model for flow matching on sequences. + """ + + def __init__(self, params: Step1XParams, args=None): + super().__init__() + + self.params = params + self.in_channels = params.in_channels + self.out_channels = params.out_channels + if params.hidden_size % params.num_heads != 0: + raise ValueError(f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}") + pe_dim = params.hidden_size // params.num_heads + if sum(params.axes_dim) != pe_dim: + raise ValueError(f"Got {params.axes_dim} but expected positional dim {pe_dim}") + self.hidden_size = params.hidden_size + self.num_heads = params.num_heads + self.pe_embedder = EmbedND(dim=pe_dim, theta=params.theta, axes_dim=params.axes_dim) + self.img_in = nn.Linear(self.in_channels, self.hidden_size, bias=True) + self.time_in = MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) + self.vector_in = MLPEmbedder(params.vec_in_dim, self.hidden_size) + self.txt_in = nn.Linear(params.context_in_dim, self.hidden_size) + + self.double_blocks = nn.ModuleList([DoubleStreamBlock( + self.hidden_size, + self.num_heads, + mlp_ratio=params.mlp_ratio, + qkv_bias=params.qkv_bias, + ) for _ in range(params.depth)]) + + self.single_blocks = nn.ModuleList([SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio) for _ in range(params.depth_single_blocks)]) + + self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels) + + self.connector = Qwen2Connector() + + # adapted from kohya definition + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + self.blocks_to_swap = None + + self.offloader_double = None + self.offloader_single = None + self.num_double_blocks = len(self.double_blocks) + self.num_single_blocks = len(self.single_blocks) + + self.disperse_loss = args is not None and args.disperse_loss + + @property + def device(self): + return next(self.parameters()).device + + @property + def dtype(self): + return next(self.parameters()).dtype + + def enable_gradient_checkpointing(self, cpu_offload: bool = False): + self.gradient_checkpointing = True + self.cpu_offload_checkpointing = cpu_offload + + self.time_in.enable_gradient_checkpointing() + self.vector_in.enable_gradient_checkpointing() + + for block in self.double_blocks + self.single_blocks: + block.enable_gradient_checkpointing(cpu_offload=cpu_offload) + + print(f"Base model: Gradient checkpointing enabled. CPU offload: {cpu_offload}") + + def disable_gradient_checkpointing(self): + self.gradient_checkpointing = False + self.cpu_offload_checkpointing = False + + self.time_in.disable_gradient_checkpointing() + self.vector_in.disable_gradient_checkpointing() + + for block in self.double_blocks + self.single_blocks: + block.disable_gradient_checkpointing() + + print("Base Model: Gradient checkpointing disabled.") + + def enable_block_swap(self, num_blocks: int, device: torch.device): + self.blocks_to_swap = num_blocks + double_blocks_to_swap = num_blocks // 2 + single_blocks_to_swap = (num_blocks - double_blocks_to_swap) * 2 + + assert double_blocks_to_swap <= self.num_double_blocks - 2 and single_blocks_to_swap <= self.num_single_blocks - 2, (f"Cannot swap more than {self.num_double_blocks - 2} double blocks and {self.num_single_blocks - 2} single blocks. " + f"Requested {double_blocks_to_swap} double blocks and {single_blocks_to_swap} single blocks.") + + self.offloader_double = custom_offloading_utils.ModelOffloader( + self.double_blocks, self.num_double_blocks, double_blocks_to_swap, device # , debug=True + ) + self.offloader_single = custom_offloading_utils.ModelOffloader( + self.single_blocks, self.num_single_blocks, single_blocks_to_swap, device # , debug=True + ) + print(f"Base model: Block swap enabled. Swapping {num_blocks} blocks, double blocks: {double_blocks_to_swap}, single blocks: {single_blocks_to_swap}.") + + def move_to_device_except_swap_blocks(self, device: torch.device): + # assume model is on cpu. do not move blocks to device to reduce temporary memory usage + if self.blocks_to_swap: + save_double_blocks = self.double_blocks + save_single_blocks = self.single_blocks + self.double_blocks = None + self.single_blocks = None + + self.to(device) + + if self.blocks_to_swap: + self.double_blocks = save_double_blocks + self.single_blocks = save_single_blocks + + def prepare_block_swap_before_forward(self): + if self.blocks_to_swap is None or self.blocks_to_swap == 0: + return + self.offloader_double.prepare_block_devices_before_forward(self.double_blocks) + self.offloader_single.prepare_block_devices_before_forward(self.single_blocks) + + @staticmethod + def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 1000.0): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + t = time_factor * t + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(t.device) + + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + if torch.is_floating_point(t): + embedding = embedding.to(t) + return embedding + + def forward( + self, + img: Tensor, + img_ids: Tensor, + txt_ids: Tensor, + timesteps: Tensor, + llm_embedding: Tensor, + t_vec: Tensor, + mask: Tensor, + ): #4068*3 ; #640*3;;640*3584 + feat = None + llm_embedding = llm_embedding.detach() + txt, y = self.connector( #->640*4096,1*768 + llm_embedding, t_vec, mask) + if img.ndim != 3 or txt.ndim != 3: + raise ValueError("Input img and txt tensors must have 3 dimensions.") + + img = self.img_in(img) #->4068*3072 + vec = self.time_in(self.timestep_embedding(timesteps, 256)) + + vec = vec + self.vector_in(y) + txt = self.txt_in(txt) + ids = torch.cat((txt_ids, img_ids), dim=1) + pe = self.pe_embedder(ids) + + if not self.blocks_to_swap: + for block in self.double_blocks: + img, txt = block(img=img, txt=txt, vec=vec, pe=pe) + + img = torch.cat((txt, img), 1) + for i, block in enumerate(self.single_blocks): + img = block(img, vec=vec, pe=pe) + if i == 9 and self.disperse_loss: + feat = img#1*6748*3072 + else: + for block_idx, block in enumerate(self.double_blocks): + self.offloader_double.wait_for_block(block_idx) + img, txt = block(img=img, txt=txt, vec=vec, pe=pe) + self.offloader_double.submit_move_blocks(self.double_blocks, block_idx) + + img = torch.cat((txt, img), 1) + + for block_idx, block in enumerate(self.single_blocks): + self.offloader_single.wait_for_block(block_idx) + img = block(img, vec=vec, pe=pe) + self.offloader_single.submit_move_blocks(self.single_blocks, block_idx) + img = img[:, txt.shape[1]:, ...] + + if self.training and self.cpu_offload_checkpointing: + img = img.to(self.device) + vec = vec.to(self.device) + + img = self.final_layer(img, vec) # (N, T, patch_size ** 2 * out_channels) + return img, feat + + +if __name__ == "__main__": + # Example usage + params = Step1XParams(in_channels=768, out_channels=768, vec_in_dim=256, context_in_dim=768, hidden_size=768, mlp_ratio=4.0, num_heads=12, depth=12, depth_single_blocks=6, axes_dim=[1, 2, 3], theta=10000, qkv_bias=True) + model = Step1XEdit(params)