jitenbot/bot/entries/jitenon.py

import re
from abc import abstractmethod
from datetime import datetime, date
from bs4 import BeautifulSoup

from bot.entries.entry import Entry
import bot.entries.expressions as Expressions


class _JitenonEntry(Entry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.expression = ""
        self.yomikata = ""
        self.definition = ""
        self.other_forms = []
        self.modified_date = date(1970, 1, 1)
        self.attribution = ""

    def get_global_identifier(self):
        return f"@{self.target.value}-{format(self.entry_id, '06')}"

    def set_page(self, page):
        soup = BeautifulSoup(page, features="html5lib")
        self.__set_modified_date(page)
        self.__set_attribution(soup)
        table = soup.find(class_="kanjirighttb")
        if table is None:
            raise ValueError("Error: table data not found in page.")
        rows = table.find("tbody").find_all("tr")
        colname = ""
        for row in rows:
            colname = row.th.text if row.th is not None else colname
            colval = self.__clean_text(row.td.text)
            self.__set_column(colname, colval)
        self._page = table.decode()

    def get_page_soup(self):
        soup = BeautifulSoup(self._page, "html5lib")
        return soup

    def get_part_of_speech_tags(self):
        # Jitenon doesn't have any
        return []

    def _get_headwords(self):
        headwords = {}
        for reading in self._get_readings():
            headwords[reading] = [self.expression]
        other_form_headwords = self._other_form_headwords()
        for reading, expressions in other_form_headwords.items():
            if reading not in headwords:
                headwords[reading] = []
            for expression in expressions:
                if expression not in headwords[reading]:
                    headwords[reading].append(expression)
        return headwords

    @abstractmethod
    def _get_column_map(self):
        pass

    def __set_modified_date(self, page):
        m = re.search(r"\"dateModified\": \"(\d{4}-\d{2}-\d{2})", page)
        if m is None:
            return
        modified_date = datetime.strptime(m.group(1), '%Y-%m-%d').date()
        self.modified_date = modified_date

    def __set_attribution(self, soup):
        attribution = soup.find(class_="copyright")
        if attribution is not None:
            self.attribution = soup.find(class_="copyright").text
        else:
            self.attribution = ""

    def __set_column(self, colname, colval):
        column_map = self._get_column_map()
        attr_name = column_map[colname]
        attr_value = getattr(self, attr_name)
        if isinstance(attr_value, str):
            setattr(self, attr_name, colval)
        elif isinstance(attr_value, list):
            if len(attr_value) == 0:
                setattr(self, attr_name, [colval])
            else:
                attr_value.append(colval)

    def _get_readings(self):
        yomikata = self.yomikata
        m = re.search(r"^[ぁ-ヿ、]+$", yomikata)
        if m:
            return [yomikata]
        m = re.search(r"^([ぁ-ヿ、]+)※", yomikata)
        if m:
            return [m.group(1)]
        m = re.search(r"^[ぁ-ヿ、]+（[ぁ-ヿ、]）[ぁ-ヿ、]+$", yomikata)
        if m:
            return Expressions.expand_abbreviation(yomikata)
        m = re.search(r"^([ぁ-ヿ、]+)（([ぁ-ヿ/\s、]+)）$", yomikata)
        if m:
            yomikatas = [m.group(1)]
            alts = m.group(2).split("/")
            for alt in alts:
                yomikatas.append(alt.strip())
            return yomikatas
        print(f"Invalid 読み方 format: {self.yomikata}\n{self}\n")
        return [""]

    def _other_form_headwords(self):
        other_form_headwords = {}
        for val in self.other_forms:
            m = re.search(r"^([^（]+)（([ぁ-ヿ、]+)）$", val)
            if not m:
                print(f"Invalid 異形 format: {val}\n{self}\n")
                continue
            expression = m.group(1)
            reading = m.group(2)
            if reading not in other_form_headwords:
                other_form_headwords[reading] = []
            if expression not in other_form_headwords[reading]:
                other_form_headwords[reading].append(expression)
        return other_form_headwords

    @staticmethod
    def __clean_text(text):
        text = text.replace("\n", "")
        text = text.replace(",", "、")
        text = text.replace(" ", "")
        text = text.strip()
        return text

    def __str__(self):
        column_map = self._get_column_map()
        colvals = [str(self.entry_id)]
        for attr_name in column_map.values():
            attr_val = getattr(self, attr_name)
            if isinstance(attr_val, str):
                colvals.append(attr_val)
            elif isinstance(attr_val, list):
                colvals.append("；".join(attr_val))
        return ",".join(colvals)


class JitenonYojiEntry(_JitenonEntry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.origin = ""
        self.kanken_level = ""
        self.category = ""
        self.related_expressions = []

    def _get_column_map(self):
        return {
            "四字熟語": "expression",
            "読み方":   "yomikata",
            "意味":     "definition",
            "異形":     "other_forms",
            "出典":     "origin",
            "漢検級":   "kanken_level",
            "場面用途": "category",
            "類義語":   "related_expressions",
        }

    def _add_variant_expressions(self, headwords):
        for expressions in headwords.values():
            Expressions.add_variant_kanji(expressions)


class JitenonKotowazaEntry(_JitenonEntry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.origin = ""
        self.example = ""
        self.related_expressions = []

    def _get_column_map(self):
        return {
            "言葉":   "expression",
            "読み方": "yomikata",
            "意味":   "definition",
            "異形":   "other_forms",
            "出典":   "origin",
            "例文":   "example",
            "類句":   "related_expressions",
        }

    def _get_headwords(self):
        if self.expression == "金棒引き・鉄棒引き":
            headwords = {
                "かなぼうひき": ["金棒引き", "鉄棒引き"]
            }
        else:
            headwords = super()._get_headwords()
        return headwords

    def _add_variant_expressions(self, headwords):
        for expressions in headwords.values():
            Expressions.add_variant_kanji(expressions)
            Expressions.add_fullwidth(expressions)


class JitenonKokugoEntry(_JitenonEntry):
    def __init__(self, target, entry_id):
        super().__init__(target, entry_id)
        self.example = ""
        self.alt_expression = ""
        self.antonym = ""
        self.attachments = ""
        self.compounds = ""
        self.related_words = ""

    def _get_column_map(self):
        return {
            "言葉":   "expression",
            "読み方": "yomikata",
            "意味":   "definition",
            "例文":   "example",
            "別表記": "alt_expression",
            "対義語": "antonym",
            "活用":   "attachments",
            "用例":   "compounds",
            "類語":   "related_words",
        }

    def _get_headwords(self):
        headwords = {}
        for reading in self.yomikata.split("・"):
            if reading not in headwords:
                headwords[reading] = []
            for expression in self.expression.split("・"):
                headwords[reading].append(expression)
            if self.alt_expression.strip() != "":
                for expression in self.alt_expression.split("・"):
                    headwords[reading].append(expression)
        return headwords

    def _add_variant_expressions(self, headwords):
        for expressions in headwords.values():
            Expressions.add_variant_kanji(expressions)
            Expressions.add_fullwidth(expressions)
            Expressions.remove_iteration_mark(expressions)
            Expressions.add_iteration_mark(expressions)