Update gallery_scraper.py

gallery_scraper.py

@@ -30,15 +30,16 @@ _VALID_FILTER = ("none", "pg13")
 
 class GalleryScraper:
     """
-    Scraper for Perchance AI Gallery.
+    Perchance AI Gallery scraper.
 
     Example:
-        scraper = GalleryScraper(pages=3, sort="top")
-        print(scraper.data)
+        result = GalleryScraper(start_page=1, pages=3, sort="top")
+        print(result.data)
     """
 
     def __init__(
         self,
+        start_page: int = 1,
         pages: int = 1,
         sort: str = "top",
         time_range: str = "all-time",
@@ -47,20 +48,25 @@ class GalleryScraper:
         timeout: int = 30,
         save: Union[bool, str] = False,
     ):
+        if start_page < 1:
+            raise ValueError("start_page must be >= 1")
+        if pages < 1:
+            raise ValueError("pages must be >= 1")
         if sort not in _VALID_SORT:
             raise ValueError(f"sort must be one of {_VALID_SORT}, got '{sort}'")
         if time_range not in _VALID_TIME:
             raise ValueError(f"time_range must be one of {_VALID_TIME}, got '{time_range}'")
         if content_filter not in _VALID_FILTER:
             raise ValueError(f"content_filter must be one of {_VALID_FILTER}, got '{content_filter}'")
-        if pages < 1:
-            raise ValueError("pages must be >= 1")
+        if concurrency < 1:
+            concurrency = 1
 
+        self.start_page = start_page
         self.pages = pages
         self.sort = sort
         self.time_range = time_range
         self.content_filter = content_filter
-        self.concurrency = max(1, concurrency)
+        self.concurrency = concurrency
         self.timeout = timeout
 
         self.data: list[dict] = []
@@ -68,27 +74,27 @@ class GalleryScraper:
         self.elapsed: float = 0.0
 
         self._log(
-            f"pages={pages}  concurrency={self.concurrency}  "
+            f"start_page={start_page}  pages={pages}  concurrency={concurrency}  "
             f"sort={sort}  time={time_range}  filter={content_filter}"
         )
         self._log("=" * 60)
 
-        start = time.time()
+        started = time.time()
         scraper = cloudscraper.create_scraper()
 
         raw_pages = self._fetch_all(scraper)
         self.data = self._parse_all(raw_pages)
         self.total = len(self.data)
 
-        self.elapsed = time.time() - start
+        self.elapsed = time.time() - started
         self._log("=" * 60)
         self._log(f"Done  |  {self.total} items  |  {self.elapsed:.2f}s")
 
         if save:
             self._save(save)
 
-    def _build_params(self, page: int) -> dict:
-        skip = page * _PER_PAGE
+    def _build_params(self, page_index: int) -> dict:
+        skip = page_index * _PER_PAGE
         params = {
             "sort": self.sort,
             "timeRange": self.time_range,
@@ -101,47 +107,52 @@ class GalleryScraper:
             params["skip"] = skip
         return params
 
-    def _fetch_one(self, scraper: cloudscraper.CloudScraper, page: int) -> tuple[int, str]:
-        skip = page * _PER_PAGE
-        self._log(f"  [→] Fetching page {page + 1}  (skip={skip}) ...")
+    def _fetch_one(
+        self,
+        scraper: cloudscraper.CloudScraper,
+        page_index: int,
+    ) -> tuple[int, str]:
+        actual_page = self.start_page + page_index
+        skip = (actual_page - 1) * _PER_PAGE
+        self._log(f"  [→] Fetching page {actual_page}  (skip={skip}) ...")
         t = time.time()
 
         try:
             resp = scraper.get(
                 _GALLERY_URL,
-                params=self._build_params(page),
+                params=self._build_params(actual_page - 1),
                 headers=_HEADERS,
                 timeout=self.timeout,
             )
         except Exception as exc:
-            self._log(f"  [✗] Page {page + 1} error: {exc}  ({time.time() - t:.2f}s)")
-            return (page, "")
+            self._log(f"  [✗] Page {actual_page} error: {exc}  ({time.time() - t:.2f}s)")
+            return (page_index, "")
 
         dt = time.time() - t
         if resp.status_code != 200:
-            self._log(f"  [✗] Page {page + 1} HTTP {resp.status_code}  ({dt:.2f}s)")
-            return (page, "")
+            self._log(f"  [✗] Page {actual_page} HTTP {resp.status_code}  ({dt:.2f}s)")
+            return (page_index, "")
 
-        self._log(f"  [✓] Page {page + 1} OK — {len(resp.text):,} chars  ({dt:.2f}s)")
-        return (page, resp.text)
+        self._log(f"  [✓] Page {actual_page} OK — {len(resp.text):,} chars  ({dt:.2f}s)")
+        return (page_index, resp.text)
 
     def _fetch_all(self, scraper: cloudscraper.CloudScraper) -> dict[int, str]:
         results: dict[int, str] = {}
 
         if self.concurrency == 1:
-            for pg in range(self.pages):
-                page, html = self._fetch_one(scraper, pg)
-                results[page] = html
+            for page_index in range(self.pages):
+                idx, html = self._fetch_one(scraper, page_index)
+                results[idx] = html
             return results
 
         with ThreadPoolExecutor(max_workers=self.concurrency) as pool:
             futures = {
-                pool.submit(self._fetch_one, scraper, pg): pg
-                for pg in range(self.pages)
+                pool.submit(self._fetch_one, scraper, page_index): page_index
+                for page_index in range(self.pages)
             }
             for future in as_completed(futures):
-                pg, html = future.result()
-                results[pg] = html
+                idx, html = future.result()
+                results[idx] = html
 
         return results
 
@@ -192,9 +203,10 @@ class GalleryScraper:
     def _parse_all(self, raw_pages: dict[int, str]) -> list[dict]:
         all_items: list[dict] = []
 
-        for pg in sorted(raw_pages.keys()):
-            parsed = self._parse_page(raw_pages[pg])
-            self._log(f"  [parse] Page {pg + 1} → {len(parsed)} items")
+        for page_index in sorted(raw_pages.keys()):
+            parsed = self._parse_page(raw_pages[page_index])
+            actual_page = self.start_page + page_index
+            self._log(f"  [parse] Page {actual_page} → {len(parsed)} items")
             all_items.extend(parsed)
 
         for idx, item in enumerate(all_items, start=1):
@@ -229,6 +241,7 @@ class GalleryScraper:
             f"GalleryScraper("
             f"total={self.total}, "
             f"pages={self.pages}, "
+            f"start_page={self.start_page}, "
             f"sort='{self.sort}', "
             f"elapsed={self.elapsed:.2f}s)"
         )