process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "66322380",
   "metadata": {},
   "source": [
    "# Filter local_add.csv to rows whose original_video exists in subject/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4877bf6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "CSV_PATH = 'OpenVE-3M/csv_files/local_add.csv'\n",
    "SUBJECT_DIR = 'OpenVE-3M/subject'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "418784f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total rows in CSV: 400498\n",
      "                                            video  \\\n",
      "0  local_add/733540474a66c15f1cbbf13c0ffc048a.mp4   \n",
      "1  local_add/e68ef344f8a8a2e997d8fde71dad9177.mp4   \n",
      "2  local_add/633eaa852e252800f8e8a6e78313cf6b.mp4   \n",
      "\n",
      "                                              prompt  \\\n",
      "0  Overlay an animated red frisbee on the edge of...   \n",
      "1  Overlay an animated modern kitchen timer onto ...   \n",
      "2  Overlay an animated digital kitchen timer onto...   \n",
      "\n",
      "                                   original_video  \n",
      "0  local_add/a3706564d18a992d8f33fd2155068aba.mp4  \n",
      "1  local_add/c4c50a63eede7c11a91e99d20faaed96.mp4  \n",
      "2  local_add/c2a6fcb84b10acff3c4db601a0efe1ae.mp4  \n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(CSV_PATH, sep=';')\n",
    "print(f'Total rows in CSV: {len(df)}')\n",
    "print(df.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "40631375",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in subject/: 6481\n"
     ]
    }
   ],
   "source": [
    "subject_files = set(os.listdir(SUBJECT_DIR))\n",
    "print(f'Files in subject/: {len(subject_files)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "409ef92e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Matching rows: 580\n",
      "                                              video  \\\n",
      "49   local_add/b3ad9626b6bfc05c78a4c46052df8230.mp4   \n",
      "347  local_add/3caae88d14d5a21e49df992f60b581b6.mp4   \n",
      "348  local_add/ec51a09b2c241d5a9b2f1895d7267c57.mp4   \n",
      "461  local_add/0ef012777ac310efa9cee99d847baaab.mp4   \n",
      "482  local_add/0367fab70a24522175e53d3c26f3454d.mp4   \n",
      "\n",
      "                                                prompt  \\\n",
      "49   Add an animated small table lamp to the desk c...   \n",
      "347  Overlay an animated pigeon on the pavement nea...   \n",
      "348  Overlay an animated wooden street bench on the...   \n",
      "461  Overlay an animated black camera tripod with a...   \n",
      "482  Overlay an animated lit birthday candle on the...   \n",
      "\n",
      "                                        original_video  \n",
      "49   global_style/b9c2b5ec245a9eaa47d7dd9821857eac.mp4  \n",
      "347  background_change/05d89575c335c3e33cc9031fdd86...  \n",
      "348  background_change/05d89575c335c3e33cc9031fdd86...  \n",
      "461  global_style/5ef04508e112f1f6733d8eb426422008.mp4  \n",
      "482     local_add/04fbe4923e36aeef135f3c9a3f029f70.mp4  \n"
     ]
    }
   ],
   "source": [
    "# Extract just the filename from the original_video path (e.g. 'local_add/abc.mp4' -> 'abc.mp4')\n",
    "df['original_video_filename'] = df['original_video'].apply(lambda p: os.path.basename(p))\n",
    "\n",
    "# Keep only rows where that filename exists in subject/\n",
    "mask = df['original_video_filename'].isin(subject_files)\n",
    "filtered = df[mask].drop(columns=['original_video_filename'])\n",
    "\n",
    "print(f'Matching rows: {len(filtered)}')\n",
    "print(filtered.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8f831e0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved 580 rows to OpenVE-3M/csv_files/local_add_in_subject.csv\n"
     ]
    }
   ],
   "source": [
    "output_path = 'OpenVE-3M/csv_files/local_add_in_subject.csv'\n",
    "filtered.to_csv(output_path, sep=';', index=False)\n",
    "print(f'Saved {len(filtered)} rows to {output_path}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c59ccec8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "flexvideo",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}