jitenbot/bot/entries/smk8.py

from bs4 import BeautifulSoup

import bot.entries.expressions as Expressions
import bot.soup as Soup
from bot.data import load_smk8_phrase_readings
from bot.entries.entry import Entry
from bot.entries.smk8_preprocess import preprocess_page


class _BaseSmk8Entry(Entry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.children = []
        self.phrases = []
        self.kanjis = []

    def get_global_identifier(self):
        parent_part = format(self.entry_id[0], '06')
        child_part = hex(self.entry_id[1]).lstrip('0x').zfill(4).upper()
        return f"@{self.target.value}-{parent_part}-{child_part}"

    def set_page(self, page):
        page = self.__decompose_subentries(page)
        self._page = page

    def get_page_soup(self):
        soup = BeautifulSoup(self._page, "xml")
        return soup

    def get_part_of_speech_tags(self):
        if self._part_of_speech_tags is not None:
            return self._part_of_speech_tags
        self._part_of_speech_tags = []
        soup = self.get_page_soup()
        headword_info = soup.find("見出要素")
        if headword_info is None:
            return self._part_of_speech_tags
        for tag in headword_info.find_all("品詞M"):
            if tag.text not in self._part_of_speech_tags:
                self._part_of_speech_tags.append(tag.text)
        return self._part_of_speech_tags

    def _add_variant_expressions(self, headwords):
        for expressions in headwords.values():
            Expressions.add_variant_kanji(expressions)
            Expressions.add_fullwidth(expressions)
            Expressions.remove_iteration_mark(expressions)
            Expressions.add_iteration_mark(expressions)

    def _find_reading(self, soup):
        midasi_kana = soup.find("見出仮名")
        reading = midasi_kana.text
        for x in [" ", "・"]:
            reading = reading.replace(x, "")
        return reading

    def _find_expressions(self, soup):
        clean_expressions = []
        for expression in soup.find_all("標準表記"):
            clean_expression = self._clean_expression(expression.text)
            clean_expressions.append(clean_expression)
        expressions = Expressions.expand_abbreviation_list(clean_expressions)
        return expressions

    def __decompose_subentries(self, page):
        soup = BeautifulSoup(page, features="xml")
        subentry_parameters = [
            [Smk8ChildEntry, ["子項目F", "子項目"], self.children],
            [Smk8PhraseEntry, ["句項目F", "句項目"], self.phrases],
            [Smk8KanjiEntry, ["造語成分項目"], self.kanjis],
        ]
        for x in subentry_parameters:
            subentry_class, tags, subentry_list = x
            for tag in tags:
                tag_soup = soup.find(tag)
                while tag_soup is not None:
                    tag_soup.name = "項目"
                    subentry_id = self.id_string_to_entry_id(tag_soup.attrs["id"])
                    self.SUBENTRY_ID_TO_ENTRY_ID[subentry_id] = self.entry_id
                    subentry = subentry_class(self.target, subentry_id)
                    page = tag_soup.decode()
                    subentry.set_page(page)
                    subentry_list.append(subentry)
                    tag_soup.decompose()
                    tag_soup = soup.find(tag)
        return soup.decode()

    @staticmethod
    def id_string_to_entry_id(id_string):
        parts = id_string.split("-")
        if len(parts) == 1:
            return (int(parts[0]), 0)
        elif len(parts) == 2:
            # subentries have a hexadecimal part
            return (int(parts[0]), int(parts[1], 16))
        else:
            raise Exception(f"Invalid entry ID: {id_string}")

    @staticmethod
    def _delete_unused_nodes(soup):
        """Remove extra markup elements that appear in the entry
        headword line which are not part of the entry headword"""
        unused_nodes = [
            "表音表記", "表外音訓マーク", "表外字マーク", "ルビG"
        ]
        for name in unused_nodes:
            Soup.delete_soup_nodes(soup, name)

    @staticmethod
    def _clean_expression(expression):
        for x in ["〈", "〉", "｛", "｝", "…", " "]:
            expression = expression.replace(x, "")
        return expression

    @staticmethod
    def _fill_alts(soup):
        for el in soup.find_all(["親見出仮名", "親見出表記"]):
            el.string = el.attrs["alt"]
        for gaiji in soup.find_all("外字"):
            gaiji.string = gaiji.img.attrs["alt"]


class Smk8Entry(_BaseSmk8Entry):
    def __init__(self, target, page_id):
        entry_id = (page_id, 0)
        super().__init__(target, entry_id)

    def set_page(self, page):
        page = preprocess_page(page)
        super().set_page(page)

    def _get_headwords(self):
        soup = self.get_page_soup()
        self._delete_unused_nodes(soup)
        self._fill_alts(soup)
        reading = self._find_reading(soup)
        expressions = []
        if soup.find("見出部").find("標準表記") is None:
            expressions.append(reading)
        for expression in self._find_expressions(soup):
            if expression not in expressions:
                expressions.append(expression)
        headwords = {reading: expressions}
        return headwords


class Smk8ChildEntry(_BaseSmk8Entry):
    def _get_headwords(self):
        soup = self.get_page_soup()
        self._delete_unused_nodes(soup)
        self._fill_alts(soup)
        reading = self._find_reading(soup)
        expressions = []
        if soup.find("子見出部").find("標準表記") is None:
            expressions.append(reading)
        for expression in self._find_expressions(soup):
            if expression not in expressions:
                expressions.append(expression)
        headwords = {reading: expressions}
        return headwords


class Smk8PhraseEntry(_BaseSmk8Entry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.__phrase_readings = load_smk8_phrase_readings()

    def get_part_of_speech_tags(self):
        # phrases do not contain these tags
        return []

    def _get_headwords(self):
        soup = self.get_page_soup()
        headwords = {}
        expressions = self._find_expressions(soup)
        readings = self._find_readings()
        for idx, expression in enumerate(expressions):
            reading = readings[idx]
            if reading in headwords:
                headwords[reading].append(expression)
            else:
                headwords[reading] = [expression]
        return headwords

    def _find_expressions(self, soup):
        self._delete_unused_nodes(soup)
        self._fill_alts(soup)
        text = soup.find("標準表記").text
        text = self._clean_expression(text)
        alternatives = Expressions.expand_smk_alternatives(text)
        expressions = []
        for alt in alternatives:
            for exp in Expressions.expand_abbreviation(alt):
                expressions.append(exp)
        return expressions

    def _find_readings(self):
        text = self.__phrase_readings[self.entry_id]
        alternatives = Expressions.expand_smk_alternatives(text)
        readings = []
        for alt in alternatives:
            for reading in Expressions.expand_abbreviation(alt):
                readings.append(reading)
        return readings


class Smk8KanjiEntry(_BaseSmk8Entry):
    def _get_headwords(self):
        soup = self.get_page_soup()
        self._delete_unused_nodes(soup)
        self._fill_alts(soup)
        reading = self.__get_parent_reading()
        expressions = self._find_expressions(soup)
        headwords = {reading: expressions}
        return headwords

    def __get_parent_reading(self):
        parent_id = self.SUBENTRY_ID_TO_ENTRY_ID[self.entry_id]
        parent = self.ID_TO_ENTRY[parent_id]
        reading = parent.get_first_reading()
        return reading