| | |
| |
|
| | from copy import deepcopy |
| |
|
| | from ultradata_math_parser.utils import * |
| | from ultradata_math_parser.parsers.base_parser import BaseParser |
| | from ultradata_math_parser.parsers.title_parser import TitleParser |
| |
|
| |
|
| | class ArticleParser(BaseParser): |
| | def __init__(self) -> None: |
| | super().__init__() |
| |
|
| | def extract(self, html="", **kwargs) -> dict: |
| | base_url = kwargs.get("base_url", "") |
| | self.process_math = kwargs.get("process_math", self.process_math) |
| | self.preserve_math_containers = kwargs.get("preserve_math_containers", self.preserve_math_containers) |
| | self.include_tables = kwargs.get("include_tables", self.include_tables) |
| | self.include_images = kwargs.get("include_images", self.include_images) |
| | html = html.replace(" ", " ").replace(" ", " ") |
| | tree = load_html(html) |
| | if tree is None: |
| | raise ValueError |
| |
|
| | title = TitleParser().process(tree) |
| |
|
| | |
| | base_href = tree.xpath("//base/@href") |
| |
|
| | if base_href and "http" in base_href[0]: |
| | base_url = base_href[0] |
| |
|
| | if "://blog.csdn.net/" in base_url: |
| | for dtree in tree.xpath('//div[@id="content_views"]//ul[@class="pre-numbering"]'): |
| | self.remove_node(dtree) |
| |
|
| | raw_tree = deepcopy(tree) |
| | working_tree = deepcopy(tree) |
| |
|
| | |
| | format_tree = self.convert_tags(working_tree, base_url=base_url) |
| | format_tree = self._remove_tables_from_tree(format_tree) |
| | format_tree = self._remove_images_from_tree(format_tree) |
| |
|
| | |
| | normal_tree = self.clean_tags(format_tree) |
| | normal_tree = self._remove_tables_from_tree(normal_tree) |
| | normal_tree = self._remove_images_from_tree(normal_tree) |
| | fallback_tree = deepcopy(normal_tree) |
| |
|
| | subtree, xp_num, drop_list = self.xp_1_5(normal_tree) |
| | if xp_num == "others": |
| | subtree, drop_list = self.prune_unwanted_sections(normal_tree) |
| | body_html = self.get_content_html(subtree, xp_num, base_url) |
| |
|
| | body_html, fallback_strategy = self.apply_fallbacks( |
| | primary_html=body_html, |
| | base_url=base_url, |
| | normal_tree=fallback_tree, |
| | raw_tree=raw_tree, |
| | ) |
| |
|
| | body_html = self._strip_tables_from_html(body_html) |
| | body_html = self._strip_images_from_html(body_html) |
| |
|
| | text_length = self._text_length_from_html(body_html) |
| |
|
| | return { |
| | "xp_num": xp_num, |
| | "drop_list": drop_list, |
| | "html": body_html, |
| | "title": title, |
| | "base_url": base_url, |
| | "fallback_strategy": fallback_strategy, |
| | "text_length": text_length, |
| | } |
| |
|
