heymenn commited on
Commit
f27076f
·
1 Parent(s): 681fc53

fix the classficiations retrieval

Browse files
Files changed (1) hide show
  1. scrap.py +16 -13
scrap.py CHANGED
@@ -75,21 +75,24 @@ async def scrap_patent_async(client: AsyncClient, patent_url: str) -> PatentScra
75
  # pub_num = soup.select_one("h2#pubnum").get_text(strip=True)
76
  # get the h2 with id ="pubnum" and extract the text
77
 
78
- # Classification codes (CPC + IPC, flat list, deduplicated by code)
79
- code_pattern = re.compile(r'[A-Z]\d{2}[A-Z]\d+/\d+')
 
 
80
  classifications = []
81
  seen_codes: set[str] = set()
82
- cls_section = soup.find("section", id="classifications")
83
- if cls_section:
84
- for item in cls_section.find_all("li"):
85
- text = item.get_text(separator=" ", strip=True)
86
- match = code_pattern.search(text)
87
- if match:
88
- code = match.group()
89
- if code not in seen_codes:
90
- seen_codes.add(code)
91
- desc = text.replace(code, "", 1).strip()
92
- classifications.append(ClassificationCode(code=code, description=desc))
 
93
 
94
  return PatentScrapResult(
95
  # publication_number=pub_num,
 
75
  # pub_num = soup.select_one("h2#pubnum").get_text(strip=True)
76
  # get the h2 with id ="pubnum" and extract the text
77
 
78
+ # Classification codes (CPC + IPC, flat list, deduplicated by code).
79
+ # Google Patents renders each code in a <span> inside a <li>. Leaf entries
80
+ # (no nested <ul>) carry exactly one code; parent entries are breadcrumbs.
81
+ leaf_code_re = re.compile(r'^[A-Z]\d{2}[A-Z]\d+/\d+$')
82
  classifications = []
83
  seen_codes: set[str] = set()
84
+ for li in soup.find_all("li"):
85
+ if li.find("ul"):
86
+ continue
87
+ span = li.find("span")
88
+ if not span:
89
+ continue
90
+ code = span.get_text(strip=True)
91
+ if leaf_code_re.match(code) and code not in seen_codes:
92
+ seen_codes.add(code)
93
+ full_text = li.get_text(separator=" ", strip=True)
94
+ desc = full_text.replace(code, "", 1).strip().lstrip("—").lstrip("-").strip()
95
+ classifications.append(ClassificationCode(code=code, description=desc))
96
 
97
  return PatentScrapResult(
98
  # publication_number=pub_num,