initial applicaiton files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,16 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Models and Engines
+*.onnx
+*.onnx.data
+*.pth
+*.engine
+
+# Videos
+*.mp4
+
+# Logs
+logs/

.huggingface.yaml

@@ -0,0 +1,3 @@
+sdk: gradio
+python_version: '3.12'
+requirements_file: requirements.txt

README.md

@@ -1,14 +1,14 @@
 ---
-title: PLFNet
-emoji: 💻
-colorFrom: pink
+title: PVSNet
+emoji: 🐢
+colorFrom: gray
 colorTo: pink
 sdk: gradio
 sdk_version: 6.3.0
 app_file: app.py
 pinned: false
 license: agpl-3.0
-short_description: Single Image to Real-Time Light Field Reconstruction Model
+short_description: Real Time View Synthesis and Light Field Reconstruction Model
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,474 @@
+import gradio as gr
+import torch
+import numpy as np
+import cv2
+import io
+import tempfile
+import base64
+from PIL import Image
+import torchvision.transforms as transforms
+import parameters as params
+from model import PLFNet
+import helperFunctions as helper
+import socket
+import os
+import json
+from huggingface_hub import hf_hub_download
+import joblib
+
+REPO_ID = "3ZadeSSG/PVSNet"
+print("Downloading/Loading checkpoints from Hugging Face Hub...")
+MODEL_FLOWERS_LOCATION = hf_hub_download(
+    repo_id=REPO_ID, 
+    filename="checkpoint_best_flowers.pth"
+)
+MODEL_STANFORD_LOCATION = hf_hub_download(
+    repo_id=REPO_ID, 
+    filename="checkpoint_best_stanford.pth"
+)
+
+DEVICE = "cpu"
+
+DATASET_CHECKPOINT_MAP = {
+    "Flowers": MODEL_FLOWERS_LOCATION,
+    "Stanford": MODEL_STANFORD_LOCATION,
+}
+
+SAMPLE_IMAGE_DIR = "./sample_images"
+SAMPLE_IMAGES = {}
+for dataset_name in ["Flowers", "Stanford"]:
+    folder = os.path.join(SAMPLE_IMAGE_DIR, dataset_name)
+    if os.path.isdir(folder):
+        images = sorted([
+            os.path.join(folder, f)
+            for f in os.listdir(folder)
+            if f.lower().endswith((".png", ".jpg", ".jpeg"))
+        ])
+        SAMPLE_IMAGES[dataset_name] = images
+
+def getPositionVector(x, y, height, width):
+    vector = torch.zeros((2, height, width), dtype=torch.float)
+    normalized_x = (x - (-0.003)) / (0.003 - (-0.003))
+    normalized_y = (y - (-0.003)) / (0.003 - (-0.003))
+    vector[0, :, :] = normalized_x
+    vector[1, :, :] = normalized_y
+    return vector
+
+def predictSingleImage(model, img, target_pose, height, width):
+    transform = transforms.Compose([
+        transforms.Resize((height, width)),
+        transforms.ToTensor()
+    ])
+    img_input = transform(img).to(DEVICE)
+    output_position = getPositionVector(target_pose[0], target_pose[1], height, width).to(DEVICE)
+    with torch.no_grad():
+        img_ = torch.cat((img_input, output_position), dim=0).unsqueeze(0).to(DEVICE)
+        img_out = model(img_).detach().cpu()
+    return img_out
+
+def generateCircularTrajectory(radius, num_frames, num_loops):
+    angles = np.linspace(0, 2 * np.pi * num_loops, num_frames * num_loops)
+    return [[radius * np.cos(angle), radius * np.sin(angle)] for angle in angles]
+
+def create_video_from_memory(frames, fps=60):
+    height, width = frames[0].shape[:2]
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    temp_video = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
+    out = cv2.VideoWriter(temp_video.name, fourcc, fps, (width, height))
+    for frame in frames:
+        out.write(frame)
+    out.release()
+    return temp_video.name
+
+def process_parallax_video(img, dataset, resolution, radius, num_frames, num_loops):
+    if img is None:
+        return None
+    checkpoint_path = DATASET_CHECKPOINT_MAP.get(dataset, DATASET_CHECKPOINT_MAP["Flowers"])
+    model = PLFNet()
+    model = helper.load_Checkpoint(checkpoint_path, model, load_cpu=True)
+    model.to(DEVICE)
+    model.eval()
+
+    height, width = (352, 512) if "352x512" in resolution else (176, 256)
+    img = img.crop((0, 0, img.width, int(img.width * (11 / 16))))
+    trajectory = generateCircularTrajectory(radius, num_frames, num_loops)
+
+    frames = []
+    for pose in trajectory:
+        output_img = predictSingleImage(model, img, pose, height, width)
+        img_np = output_img.squeeze(0).permute(1, 2, 0).numpy()
+        img_np = (img_np * 255).astype(np.uint8)
+        img_bgr = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
+        frames.append(img_bgr)
+
+    return create_video_from_memory(frames)
+
+def generate_lf_raw_frames(img, dataset, resolution):
+    if img is None:
+        return None, "Please upload an image first."
+        
+    checkpoint_path = DATASET_CHECKPOINT_MAP.get(dataset, DATASET_CHECKPOINT_MAP["Flowers"])
+    model = PLFNet()
+    model = helper.load_Checkpoint(checkpoint_path, model, load_cpu=True)
+    model.to(DEVICE)
+    model.eval()
+
+    height, width = (352, 512) if "352x512" in resolution else (176, 256)
+    img = img.crop((0, 0, img.width, int(img.width * (11 / 16))))
+    
+    frames_b64 = []
+    
+    for i in range(-3, 4):
+        for j in range(-3, 4):
+            pose = [i * 0.001, j * 0.001]
+            out = predictSingleImage(model, img, pose, height, width)
+            img_np = out.squeeze(0).permute(1, 2, 0).numpy()
+            img_np = (img_np * 255).astype(np.uint8)
+            img_bgr = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
+            _, buffer = cv2.imencode('.jpg', img_bgr, [cv2.IMWRITE_JPEG_QUALITY, 80])
+            b64_str = base64.b64encode(buffer).decode('utf-8')
+            frames_b64.append(f"data:image/jpeg;base64,{b64_str}")
+            
+    import json
+    return json.dumps(frames_b64), "Light Field generated! You can now adjust Focus and Aperture, and move your mouse over the image."
+
+
+
+html_code = """
+<iframe id="lf-iframe" style="width: 100%; max-width: 800px; aspect-ratio: 512/352; border: 2px dashed #ccc; display: block; margin: 0 auto; background: #222;" srcdoc="
+<html>
+<head>
+  <script src='https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js'></script>
+  <script src='https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js'></script>
+  <style> body { margin: 0; overflow: hidden; background: #222; } canvas { display: block; width: 100%; height: 100%; } #placeholder { color: #888; font-family: sans-serif; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); pointer-events: none; } #debug { position: absolute; top: 0; left: 0; color: lime; font-family: monospace; padding: 10px; pointer-events: none; z-index: 9999; } </style>
+</head>
+<body>
+  <div id='placeholder'>Generated Light Field will appear here</div>
+  <div id='debug'></div>
+  <script>
+    function logDebug(msg) {
+        document.getElementById('debug').innerHTML += msg + '<br>';
+    }
+
+    const vertexShader = `
+      out vec2 vSt;
+      out vec2 vUv;
+      void main() {
+          vec3 posToCam = cameraPosition - position;
+          vec3 nDir = normalize(posToCam);
+          float zRatio = posToCam.z / nDir.z;
+          vec3 uvPoint = zRatio * nDir;
+          vUv = uvPoint.xy + 0.5;
+          vUv.x = 1.0 - vUv.x;
+          vSt = uv;
+          vSt.x = 1.0 - vSt.x;
+          gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+      }
+    `;
+
+    const fragmentShader = `
+      precision highp sampler2DArray;
+      uniform sampler2DArray field;
+      uniform vec2 camArraySize;
+      uniform float aperture;
+      uniform float focus;
+      in vec2 vSt;
+      in vec2 vUv;
+      out vec4 fragColor;
+      
+      void main() {
+          vec4 color = vec4(0.0);
+          float colorCount = 0.0;
+          if (vUv.x < 0.0 || vUv.x > 1.0 || vUv.y < 0.0 || vUv.y > 1.0) {
+              discard;
+          }
+          for (float i = 0.0; i < 7.0; i++) {
+              for (float j = 0.0; j < 7.0; j++) {
+                  float dx = i - (vSt.x * camArraySize.x - 0.5);
+                  float dy = j - (vSt.y * camArraySize.y - 0.5);
+                  float sqDist = dx * dx + dy * dy;
+                  if (sqDist <= aperture + 0.001) {
+                      float camOff = i + camArraySize.x * j;
+                      vec2 focOff = vec2(dx, dy) * focus;
+                      color += texture(field, vec3(vUv + focOff, camOff));
+                      colorCount++;
+                  }
+              }
+          }
+          fragColor = vec4(color.rgb / max(colorCount, 1.0), 1.0);
+      }
+    `;
+
+    let scene, camera, renderer, planeMat, fieldTexture, controls;
+    let camsX = 7, camsY = 7;
+    let reqFrame;
+
+    function initScene() {
+        if(scene) return;
+        scene = new THREE.Scene();
+        camera = new THREE.PerspectiveCamera(30, window.innerWidth / window.innerHeight, 0.1, 100);
+        camera.position.set(0, 0, 2);
+        camera.lookAt(0, 0, -2);
+
+        const canvas = document.createElement('canvas');
+        const context = canvas.getContext('webgl2', { antialias: true });
+        if (!context) logDebug('WebGL2 not supported!');
+        renderer = new THREE.WebGLRenderer({ canvas: canvas, context: context });
+        renderer.setSize(window.innerWidth, window.innerHeight);
+        document.body.appendChild(renderer.domElement);
+
+        controls = new THREE.OrbitControls(camera, renderer.domElement);
+        controls.enableDamping = true;
+        controls.target.set(0, 0, -2);
+        controls.panSpeed = 2;
+
+        window.addEventListener('resize', () => {
+            camera.aspect = window.innerWidth / window.innerHeight;
+            camera.updateProjectionMatrix();
+            renderer.setSize(window.innerWidth, window.innerHeight);
+        });
+    }
+
+    function render() {
+        if (controls) controls.update();
+        if (renderer && scene && camera) {
+            renderer.render(scene, camera);
+        }
+        reqFrame = requestAnimationFrame(render);
+    }
+
+    window.addEventListener('message', async function(e) {
+        if (e.data.type === 'update_frames') {
+            document.getElementById('placeholder').innerText = 'Loading textures...';
+            document.getElementById('placeholder').style.display = 'block';
+            
+            const frames = e.data.frames;
+            if (!frames || frames.length !== 49) {
+                logDebug('Error: Invalid frames');
+                return;
+            }
+            
+            try {
+                initScene();
+                if (controls) {
+                    controls.rotateSpeed = e.data.sensitivity;
+                    controls.panSpeed = e.data.sensitivity * 2;
+                }
+                
+                let resX, resY;
+                const allBuffer = [];
+                
+                for(let i=0; i<49; i++) {
+                    const img = new Image();
+                    await new Promise((resolve, reject) => { 
+                        img.onload = resolve; 
+                        img.onerror = reject;
+                        img.src = frames[i]; 
+                    });
+                    if(i===0) { resX = img.width; resY = img.height; }
+                    const cvs = document.createElement('canvas');
+                    cvs.width = resX; cvs.height = resY;
+                    const ctx = cvs.getContext('2d', { willReadFrequently: true });
+                    ctx.drawImage(img, 0, 0);
+                    const data = ctx.getImageData(0, 0, resX, resY).data;
+                    allBuffer.push(data);
+                }
+                
+                const totalBuffer = new Uint8Array(resX * resY * 4 * 49);
+                for(let i=0; i<49; i++) {
+                    totalBuffer.set(allBuffer[i], i * resX * resY * 4);
+                }
+
+                if(planeMat) {
+                    scene.remove(scene.children[0]);
+                    planeMat.dispose();
+                }
+
+                fieldTexture = new THREE.DataTexture2DArray(totalBuffer, resX, resY, 49);
+                fieldTexture.format = THREE.RGBAFormat;
+                fieldTexture.type = THREE.UnsignedByteType;
+                fieldTexture.needsUpdate = true;
+                
+                planeMat = new THREE.ShaderMaterial({
+                    uniforms: {
+                        field: { value: fieldTexture },
+                        camArraySize: { value: new THREE.Vector2(camsX, camsY) },
+                        aperture: { value: e.data.aperture },
+                        focus: { value: e.data.focus }
+                    },
+                    vertexShader: vertexShader,
+                    fragmentShader: fragmentShader,
+                    side: THREE.DoubleSide,
+                    glslVersion: THREE.GLSL3
+                });
+                
+                const planeGeo = new THREE.PlaneGeometry(camsX * 0.1, camsY * 0.1, camsX, camsY);
+                const plane = new THREE.Mesh(planeGeo, planeMat);
+                
+                // Scale to correct aspect ratio and increase display size (doesn't break shader parallax logic)
+                const aspect = resX / resY;
+                plane.scale.set(aspect * 2.0, 2.0, 1.0);
+                
+                plane.position.z = -2;
+                scene.add(plane);
+                
+                document.getElementById('placeholder').style.display = 'none';
+                if(!reqFrame) render(); // Start animation loop
+            } catch (err) {
+                logDebug('Error: ' + err.message);
+            }
+        }
+        else if (e.data.type === 'update_sensitivity') {
+            if (controls) {
+                controls.rotateSpeed = e.data.value;
+                controls.panSpeed = e.data.value * 2; // pan is naturally slower
+            }
+        }
+        else if (e.data.type === 'reset_camera') {
+            camera.position.set(0, 0, 2);
+            camera.up.set(0, 1, 0);
+            if (controls) {
+                controls.target.set(0, 0, -2);
+                controls.update();
+            }
+        }
+        else if (e.data.type === 'update_aperture') {
+            if(planeMat) planeMat.uniforms.aperture.value = e.data.value;
+        }
+    });
+  </script>
+</body>
+</html>
+"></iframe>
+"""
+
+def load_sample_image(image_path, dataset_name):
+    img = Image.open(image_path)
+    return img, dataset_name
+
+with gr.Blocks(title="PVSNet/PLFNet", theme="default") as demo:
+    gr.Markdown("""
+    # PVSNet: Real-Time Position-Aware View Synthesis from Single-View Input
+    * Upload a single Lytro image and get a mini parallax video showing capabilities of the light field reconstruction model from our works PVSNet and PLFNet.
+    **Note** Huggingface demo is running on CPU, so the inference speed will be slow. It might take around 2 minutes for full LF reconstruction or video generation.
+    ### Head to our [Project Page](https://realistic3d-miun.github.io/PVSNet/) for more details about the models.
+    """)
+
+    with gr.Row():
+        img_input = gr.Image(type="pil", label="Upload Image")
+        with gr.Column():
+            dataset = gr.Dropdown(
+                choices=["Flowers", "Stanford"],
+                value="Flowers",
+                label="Model Checkpoint (Dataset)"
+            )
+            resolution = gr.Dropdown(["352x512 (Slow)", "176x256 (Fast)"], value="352x512 (Slow)", label="Resolution")
+    
+    with gr.Tabs():
+        with gr.Tab("Parallax Video"):
+            with gr.Row():
+                with gr.Column():
+                    radius = gr.Slider(0.0006, 0.006, value=0.003, label="Radius")
+                    num_frames = gr.Slider(10, 100, value=60, step=10, label="Number of Frames")
+                    num_loops = gr.Slider(1, 10, value=4, step=1, label="Number of Loops")
+                    generate_vid_btn = gr.Button("Generate Video", variant="primary")
+                video_output = gr.Video(label="Generated Video", height=352)
+
+            generate_vid_btn.click(
+                fn=process_parallax_video,
+                inputs=[img_input, dataset, resolution, radius, num_frames, num_loops],
+                outputs=video_output,
+            )
+
+        with gr.Tab("Interactive Light Field"):
+            with gr.Row():
+                with gr.Column():
+                    generate_lf_btn = gr.Button("Generate Light Field Data", variant="primary")
+                    lf_status = gr.Textbox(label="Status", interactive=False, value="Awaiting generation...")
+                    
+                    gr.Markdown("### Rendering Controls\nAdjust these parameters and use your mouse to navigate the Light Field (Left Click: Rotate, Right Click: Pan, Scroll: Zoom).")
+                    with gr.Row():
+                        sensitivity = gr.Slider(0.1, 3.0, value=1.0, step=0.1, label="Mouse Sensitivity")
+                        aperture = gr.Slider(0, 4.5, value=2.2, step=0.1, label="Aperture")
+                    reset_btn = gr.Button("Reset Camera")
+                
+                with gr.Column():
+                    gr.HTML(html_code)
+            
+            # Hidden text box to transfer JSON data to frontend
+            b64_frames_state = gr.Textbox(visible=False, elem_id="lf_data_bridge")
+
+            # JS function string to update frames in iframe
+            update_js = """(val, a, s) => { 
+                if (val) {
+                    const frames = JSON.parse(val);
+                    const iframe = document.getElementById('lf-iframe');
+                    if (iframe && iframe.contentWindow) {
+                        iframe.contentWindow.postMessage({
+                            type: 'update_frames', 
+                            frames: frames,
+                            aperture: a,
+                            focus: 0,
+                            sensitivity: s
+                        }, '*');
+                    }
+                }
+            }"""
+
+            # Step 1: Generate Raw Light Field (7x7 array)
+            generate_lf_btn.click(
+                fn=generate_lf_raw_frames,
+                inputs=[img_input, dataset, resolution],
+                outputs=[b64_frames_state, lf_status]
+            ).then( # Step 2: Run JS to update frontend
+                fn=None,
+                inputs=[b64_frames_state, aperture, sensitivity],
+                outputs=None,
+                js=update_js
+            )
+
+            # Sliders update iframe instantly via postMessage (no Python execution needed!)
+            sensitivity.change(
+                fn=None,
+                inputs=[sensitivity],
+                outputs=None,
+                js="(s) => { const iframe = document.getElementById('lf-iframe'); if (iframe && iframe.contentWindow) iframe.contentWindow.postMessage({type: 'update_sensitivity', value: s}, '*'); }"
+            )
+            
+            aperture.change(
+                fn=None,
+                inputs=[aperture],
+                outputs=None,
+                js="(a) => { const iframe = document.getElementById('lf-iframe'); if (iframe && iframe.contentWindow) iframe.contentWindow.postMessage({type: 'update_aperture', value: a}, '*'); }"
+            )
+
+            reset_btn.click(
+                fn=None,
+                inputs=None,
+                outputs=None,
+                js="() => { const iframe = document.getElementById('lf-iframe'); if (iframe && iframe.contentWindow) iframe.contentWindow.postMessage({type: 'reset_camera'}, '*'); }"
+            )
+
+    gr.Markdown("### Example Images: Click to Load")
+    for dataset_name, images in SAMPLE_IMAGES.items():
+        with gr.Accordion(f"📂 {dataset_name} Samples", open=(dataset_name == "Flowers")):
+            for row_start in range(0, len(images), 3):
+                row_images = images[row_start : row_start + 3]
+                with gr.Row():
+                    for img_path in row_images:
+                        label = os.path.splitext(os.path.basename(img_path))[0]
+                        sample_img = gr.Image(
+                            img_path,
+                            label=label,
+                            height=150,
+                            interactive=False,
+                            show_label=True,
+                        )
+                        sample_img.select(
+                            fn=lambda path=img_path, ds=dataset_name: load_sample_image(path, ds),
+                            inputs=[],
+                            outputs=[img_input, dataset],
+                        )
+
+if __name__ == "__main__":
+    demo.launch()

helperFunctions.py

@@ -0,0 +1,26 @@
+import torch
+import os
+import torch.nn.functional as F
+
+def save_checkpoint(model, filelocation, save_parallel = True):
+    if save_parallel:
+        torch.save(model.module.state_dict(), filelocation)
+    else:
+        torch.save(model.state_dict(), filelocation)
+
+def load_Checkpoint(fileLocation,model, load_cpu=False):
+    if load_cpu:
+        model.load_state_dict(torch.load(fileLocation,map_location=lambda storage, loc: storage))
+    else:
+        model.load_state_dict(torch.load(fileLocation))
+    return model
+
+def writeLog(logList, filename):
+    with open(filename, 'w') as outfile:
+        outfile.write("\n".join(logList))
+
+
+def kl_loss(mu, logvar):
+    return -0.5 * (1 + logvar - mu.pow(2) - logvar.exp()).mean()        
+
+

model.py

@@ -0,0 +1,263 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import warnings
+warnings.filterwarnings("ignore")
+import torchvision
+import parameters as params
+
+def getLinearLayer(in_feat, out_feat, activation=nn.ReLU(True)):
+    return nn.Sequential(
+        nn.Linear(in_features=in_feat, out_features=out_feat, bias=True),
+        activation
+    )
+
+def getConvLayer(in_channel,out_channel,stride=1,padding=1,activation=nn.ReLU()):
+    return nn.Sequential(nn.Conv2d(in_channel, 
+                    out_channel,
+                    kernel_size=3,
+                    stride=stride,
+                    padding=padding,
+                    padding_mode='reflect'),
+                    activation)
+
+def getConvTransposeLayer(in_channel, out_channel,kernel=3,stride=1,padding=1,activation=nn.ReLU()):
+    return nn.Sequential(nn.ConvTranspose2d(in_channel,
+                                            out_channel,
+                                            kernel_size = kernel,
+                                            stride=stride,
+                                            padding=padding),
+                                            activation)
+
+
+
+class Flatten(nn.Module):
+    def forward(self, input):
+        return input.view(input.size(0), -1)
+
+class UnFlatten(nn.Module):
+    def forward(self, input, size=1):
+        return input.view(input.size(0), 1, params.params_height//16, params.params_width//16)
+
+class ResidualBlock(nn.Module):
+    def __init__(self, in_channels, out_channels, stride=1):
+        super(ResidualBlock, self).__init__()
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.relu = nn.ReLU()
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)
+        self.stride = stride
+
+        self.shortcut = nn.Sequential()
+        if stride != 1 or in_channels != out_channels:
+            self.shortcut = nn.Sequential(
+                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),
+                nn.BatchNorm2d(out_channels)
+            )
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+
+        out = out + self.shortcut(residual)
+        out = self.relu(out)
+        return out
+
+class MLPEncoder(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+        self.flat = Flatten()
+        self.layer1 = getLinearLayer((params.params_height//8)*(params.params_width//8)*2, 1024)
+        self.layer2 = getLinearLayer(1024, 512)
+        self.layer3 = getLinearLayer(512, (params.params_height//16)*(params.params_width//16))
+        self.unflat = UnFlatten()
+        self.up_layer1 = nn.Upsample(scale_factor=2, mode='nearest') 
+        self.up_layer2 = nn.Upsample(scale_factor=2, mode='nearest')
+        self.up_layer3 = nn.Upsample(scale_factor=2, mode='nearest')
+        self.up_layer4 = nn.Upsample(scale_factor=2, mode='nearest')
+
+    def forward(self, x):
+        x = self.flat(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+
+        x = self.unflat(x)
+
+        x = self.up_layer1(x)
+        x = self.up_layer2(x)
+        x = self.up_layer3(x)
+        x = self.up_layer4(x)
+        return x
+    
+class UpperEncoder(nn.Module):
+    def __init__(self):
+        super().__init__()
+        model = torchvision.models.resnet152(pretrained=False)
+        layers = list(model.children())
+        self.ResNetEncoder = torch.nn.Sequential(*layers[:5].copy())
+        del model
+
+    def forward(self, x):
+        x1 = x[:, 0:3, :, :]
+        x1 = self.ResNetEncoder(x1)
+        return x1
+
+    def apply_resnet_encoder(self, x):
+        x1 = x[:, 0:3, :, :]
+        x1 = self.ResNetEncoder(x1)
+        return x1
+
+class LowerEncoder(nn.Module):
+    def __init__(self,total_image_input=1):
+        super().__init__()
+        self.encoder_pre = ResidualBlock((total_image_input*3)+2, 20)
+        self.encoder_layer1 = ResidualBlock(20, 30)
+        self.encoder_layer2 = ResidualBlock(30, 50)
+
+        self.encoder_layer3 = nn.Sequential(
+            ResidualBlock(50, 100),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        self.encoder_layer4 = ResidualBlock(100, 200)
+        self.encoder_layer5 = nn.Sequential(
+            ResidualBlock(200, 400),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        self.encoder_layer6 = ResidualBlock(400, 600)
+        self.encoder_layer7 = nn.Sequential(
+            ResidualBlock(600, 800),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        self.encoder_layer8 = ResidualBlock(800, 1000)
+        self.encoder_layer9 = nn.Sequential(
+            ResidualBlock(1000, 1200),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+                                
+        self.encoder_layer10 = ResidualBlock(1200, 1400)
+        self.encoder_layer11 = ResidualBlock(1400, 1600)
+        
+    def forward(self, x):
+        x = self.encoder_pre(x)
+        x = self.encoder_layer1(x)
+        x = self.encoder_layer2(x)
+        skip1 = self.encoder_layer3(x)
+        
+        x = self.encoder_layer4(skip1)
+        skip2 = self.encoder_layer5(x)
+
+        x = self.encoder_layer6(skip2)
+        skip3 = self.encoder_layer7(x)
+
+        x = self.encoder_layer8(skip3)
+        skip4 = self.encoder_layer9(x)
+
+        x = self.encoder_layer10(skip4)
+        x = self.encoder_layer11(x)
+
+        return x, [skip1, skip2, skip3, skip4]
+
+class MergeDecoder(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+        self.decoder_layer1 = ResidualBlock(1600, 1400)
+        self.decoder_layer2 = ResidualBlock(1400, 1200)
+        self.decoder_layer3 = ResidualBlock(1200, 1000)
+        
+        self.decoder_layer4 = nn.Sequential(
+            nn.ConvTranspose2d(1000, 800, 2, stride=2, padding=0), 
+            nn.ReLU(True)
+        )
+        self.decoder_layer5 = ResidualBlock(800, 600)
+    
+        self.decoder_layer6 = nn.Sequential(
+            nn.ConvTranspose2d(600, 400, 2, stride=2, padding=0), 
+            nn.ReLU(True)
+        )
+        self.decoder_layer7 = ResidualBlock(400, 200)
+
+        self.decoder_layer8 = nn.Sequential(
+            nn.ConvTranspose2d(200, 100, 2, stride=2, padding=0), 
+            nn.ReLU(True)
+        )
+        self.decoder_layer9 = ResidualBlock(100, 100)
+
+        self.decoder_layer10 = nn.Sequential(
+            nn.ConvTranspose2d(100, 100, 2, stride=2, padding=0), 
+            nn.ReLU(True)
+        )
+        self.decoder_layer11 = ResidualBlock(100, 100)
+        self.decoder_layer12 = ResidualBlock(100, 50)
+        self.decoder_layer13 = ResidualBlock(50, 40)
+        self.decoder_layer14 = ResidualBlock(40, 20)
+        self.decoder_layer15 = nn.Sequential(
+            nn.Conv2d(20, 8, 3, stride=1, padding=1),
+            nn.Sigmoid()
+        )
+        self.decoder_layer16 = nn.Sequential(
+            nn.Conv2d(8, 3, 3, stride=1, padding=1),
+            nn.Sigmoid()
+        )
+        
+    def forward(self, x, lower_skip_list, upper_skip_list):
+        x = self.decoder_layer1(x)
+        x = self.decoder_layer2(x)
+        x = x + lower_skip_list[3] + upper_skip_list[1]
+
+        x = self.decoder_layer3(x)
+        x = self.decoder_layer4(x)
+        x = x + lower_skip_list[2] + upper_skip_list[0]
+        
+        x = self.decoder_layer5(x)
+        x = self.decoder_layer6(x)
+        x = x + lower_skip_list[1]
+
+        x = self.decoder_layer7(x)
+        x = self.decoder_layer8(x)
+        x = x + lower_skip_list[0]
+
+        x = self.decoder_layer9(x)
+        x = self.decoder_layer10(x)
+        x = self.decoder_layer11(x)
+        x = self.decoder_layer12(x)
+        x = self.decoder_layer13(x)
+        x = self.decoder_layer14(x)
+        x = self.decoder_layer15(x)
+        x = self.decoder_layer16(x)        
+        return x
+
+class PLFNet(nn.Module):
+    def __init__(self,total_image_input=1):
+        super().__init__()
+        self.upper_encoder = UpperEncoder()
+        self.lower_encoder = LowerEncoder(total_image_input)
+        self.merge_decoder = MergeDecoder()
+
+        self.upper_encoder_extra_1 = nn.Sequential(
+            ResidualBlock(256, 800),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        self.upper_encoder_extra_2 = nn.Sequential(
+            ResidualBlock(800, 1200),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+    def forward(self, x):
+        upper_features_1 = self.upper_encoder.apply_resnet_encoder(x)
+        upper_features_1 = self.upper_encoder_extra_1(upper_features_1)
+        upper_features_2 = self.upper_encoder_extra_2(upper_features_1)
+        
+        lower_feature, skip_list = self.lower_encoder(x)
+        merged_feature = self.merge_decoder(lower_feature, skip_list, [upper_features_1, upper_features_2])
+        
+        return merged_feature

parameters.py

@@ -0,0 +1,25 @@
+import os
+
+params_width = 512
+params_height = 352
+
+TRAIN_LOCATION = "./lf_train.txt"
+VALIDATION_LOCATION = "./lf_validate.txt"
+TEST_LOCATION = "./lf_test.txt"
+LOG_FILE_LOCATION = "./logs/training_log_0.txt"
+CHECKPOINT_LOCATION = "./checkpoint/"
+RESUME_CHECKPOINT_LOCATION = "./checkpoint/checkpoint_best.pth"
+START_CHECKPOINT_LOCATION = "./checkpoint/checkpoint_init.pth"
+DEVICE = "cpu"
+
+BATCH_SIZE = 16
+LEARNING_RATE = 0.0001
+NUM_EPOCHS = 150
+START_EPOCH = 0
+PRINT_INTERVAL = 20
+
+os.makedirs("./logs",exist_ok=True)
+os.makedirs("./checkpoint",exist_ok=True)
+os.makedirs("./output",exist_ok=True)
+
+

requirements.txt

@@ -0,0 +1,18 @@
+numpy
+torch==2.9.1
+torchvision==0.24.1
+pytorch-msssim==1.0.0
+pytorchvideo==0.1.5
+gradio==6.2.0
+gradio_client==2.0.2
+opencv-python==4.6.0.66
+pillow==10.4.0
+pillow_heif==0.15.0
+matplotlib==3.10.8
+matplotlib-inline==0.1.6
+tqdm==4.65.0
+moviepy==1.0.3
+scikit-image==0.26.0
+scikit-learn==1.8.0
+scipy==1.11.4
+random-fourier-features-pytorch