232 lines
8.3 KiB
Python
232 lines
8.3 KiB
Python
from bs4 import BeautifulSoup
|
||
|
||
import bot.entries.expressions as Expressions
|
||
import bot.soup as Soup
|
||
from bot.data import load_phrase_readings
|
||
from bot.data import load_daijirin2_kana_abbreviations
|
||
from bot.entries.entry import Entry
|
||
from bot.entries.daijirin2_preprocess import preprocess_page
|
||
|
||
|
||
class _BaseDaijirin2Entry(Entry):
|
||
def __init__(self, target, entry_id):
|
||
super().__init__(target, entry_id)
|
||
self.children = []
|
||
self.phrases = []
|
||
self._kana_abbreviations = load_daijirin2_kana_abbreviations()
|
||
|
||
def get_global_identifier(self):
|
||
parent_part = format(self.entry_id[0], '06')
|
||
child_part = hex(self.entry_id[1]).lstrip('0x').zfill(4).upper()
|
||
return f"@{self.target.value}-{parent_part}-{child_part}"
|
||
|
||
def set_page(self, page):
|
||
page = self.__decompose_subentries(page)
|
||
self._page = page
|
||
|
||
def get_page_soup(self):
|
||
soup = BeautifulSoup(self._page, "xml")
|
||
return soup
|
||
|
||
def get_part_of_speech_tags(self):
|
||
if self._part_of_speech_tags is not None:
|
||
return self._part_of_speech_tags
|
||
self._part_of_speech_tags = []
|
||
soup = self.get_page_soup()
|
||
for pos_group in soup.find_all("品詞G"):
|
||
if pos_group.parent.name == "大語義":
|
||
self._set_part_of_speech_tags(pos_group)
|
||
return self._part_of_speech_tags
|
||
|
||
def _set_part_of_speech_tags(self, el):
|
||
pos_names = ["品詞", "品詞活用", "品詞行", "用法"]
|
||
for child in el.children:
|
||
if child.name is not None:
|
||
self._set_part_of_speech_tags(child)
|
||
continue
|
||
pos = str(child)
|
||
if el.name not in pos_names:
|
||
continue
|
||
elif pos in ["［", "］"]:
|
||
continue
|
||
elif pos in self._part_of_speech_tags:
|
||
continue
|
||
else:
|
||
self._part_of_speech_tags.append(pos)
|
||
|
||
def _get_regular_headwords(self, soup):
|
||
self._fill_alts(soup)
|
||
reading = soup.find("見出仮名").text
|
||
expressions = []
|
||
for el in soup.find_all("標準表記"):
|
||
expression = self._clean_expression(el.text)
|
||
if "—" in expression:
|
||
kana_abbrs = self._kana_abbreviations[self.entry_id]
|
||
for abbr in kana_abbrs:
|
||
expression = expression.replace("—", abbr, 1)
|
||
expressions.append(expression)
|
||
expressions = Expressions.expand_abbreviation_list(expressions)
|
||
if len(expressions) == 0:
|
||
expressions.append(reading)
|
||
headwords = {reading: expressions}
|
||
return headwords
|
||
|
||
def _add_variant_expressions(self, headwords):
|
||
for expressions in headwords.values():
|
||
Expressions.add_variant_kanji(expressions)
|
||
Expressions.add_fullwidth(expressions)
|
||
Expressions.remove_iteration_mark(expressions)
|
||
Expressions.add_iteration_mark(expressions)
|
||
|
||
def __decompose_subentries(self, page):
|
||
soup = BeautifulSoup(page, features="xml")
|
||
subentry_parameters = [
|
||
[Daijirin2ChildEntry, ["子項目"], self.children],
|
||
[Daijirin2PhraseEntry, ["句項目"], self.phrases],
|
||
]
|
||
for x in subentry_parameters:
|
||
subentry_class, tags, subentry_list = x
|
||
for tag in tags:
|
||
tag_soup = soup.find(tag)
|
||
while tag_soup is not None:
|
||
tag_soup.name = "項目"
|
||
subentry_id = self.id_string_to_entry_id(tag_soup.attrs["id"])
|
||
self.SUBENTRY_ID_TO_ENTRY_ID[subentry_id] = self.entry_id
|
||
subentry = subentry_class(self.target, subentry_id)
|
||
page = tag_soup.decode()
|
||
subentry.set_page(page)
|
||
subentry_list.append(subentry)
|
||
tag_soup.decompose()
|
||
tag_soup = soup.find(tag)
|
||
return soup.decode()
|
||
|
||
@staticmethod
|
||
def id_string_to_entry_id(id_string):
|
||
parts = id_string.split("-")
|
||
if len(parts) == 1:
|
||
return (int(parts[0]), 0)
|
||
elif len(parts) == 2:
|
||
# subentries have a hexadecimal part
|
||
return (int(parts[0]), int(parts[1], 16))
|
||
else:
|
||
raise Exception(f"Invalid entry ID: {id_string}")
|
||
|
||
@staticmethod
|
||
def _delete_unused_nodes(soup):
|
||
"""Remove extra markup elements that appear in the entry
|
||
headword line which are not part of the entry headword"""
|
||
unused_nodes = [
|
||
"漢字音logo", "活用分節", "連語句活用分節", "語構成",
|
||
"表外字マーク", "表外字マーク", "ルビG"
|
||
]
|
||
for name in unused_nodes:
|
||
Soup.delete_soup_nodes(soup, name)
|
||
|
||
@staticmethod
|
||
def _clean_expression(expression):
|
||
for x in ["〈", "〉", "《", "》", " "]:
|
||
expression = expression.replace(x, "")
|
||
return expression
|
||
|
||
@staticmethod
|
||
def _fill_alts(soup):
|
||
for gaiji in soup.find_all(class_="gaiji"):
|
||
if gaiji.name == "img" and gaiji.has_attr("alt"):
|
||
gaiji.name = "span"
|
||
gaiji.string = gaiji.attrs["alt"]
|
||
|
||
|
||
class Daijirin2Entry(_BaseDaijirin2Entry):
|
||
def __init__(self, target, page_id):
|
||
entry_id = (page_id, 0)
|
||
super().__init__(target, entry_id)
|
||
|
||
def set_page(self, page):
|
||
page = preprocess_page(page)
|
||
super().set_page(page)
|
||
|
||
def _get_headwords(self):
|
||
soup = self.get_page_soup()
|
||
self._delete_unused_nodes(soup)
|
||
if soup.find("漢字見出") is not None:
|
||
headwords = self._get_kanji_headwords(soup)
|
||
elif soup.find("略語G") is not None:
|
||
headwords = self._get_acronym_headwords(soup)
|
||
else:
|
||
headwords = self._get_regular_headwords(soup)
|
||
return headwords
|
||
|
||
def _get_kanji_headwords(self, soup):
|
||
readings = []
|
||
for el in soup.find_all("漢字音"):
|
||
hira = Expressions.kata_to_hira(el.text)
|
||
readings.append(hira)
|
||
if soup.find("漢字音") is None:
|
||
readings.append("")
|
||
expressions = []
|
||
for el in soup.find_all("漢字見出"):
|
||
expressions.append(el.text)
|
||
headwords = {}
|
||
for reading in readings:
|
||
headwords[reading] = expressions
|
||
return headwords
|
||
|
||
def _get_acronym_headwords(self, soup):
|
||
expressions = []
|
||
for el in soup.find_all("略語"):
|
||
expression_parts = []
|
||
for part in el.find_all(["欧字", "和字"]):
|
||
expression_parts.append(part.text)
|
||
expression = "".join(expression_parts)
|
||
expressions.append(expression)
|
||
headwords = {"": expressions}
|
||
return headwords
|
||
|
||
|
||
class Daijirin2ChildEntry(_BaseDaijirin2Entry):
|
||
def _get_headwords(self):
|
||
soup = self.get_page_soup()
|
||
self._delete_unused_nodes(soup)
|
||
headwords = self._get_regular_headwords(soup)
|
||
return headwords
|
||
|
||
|
||
class Daijirin2PhraseEntry(_BaseDaijirin2Entry):
|
||
def get_part_of_speech_tags(self):
|
||
# phrases do not contain these tags
|
||
return []
|
||
|
||
def _get_headwords(self):
|
||
soup = self.get_page_soup()
|
||
headwords = {}
|
||
expressions = self._find_expressions(soup)
|
||
readings = self._find_readings()
|
||
for idx, expression in enumerate(expressions):
|
||
reading = readings[idx]
|
||
if reading in headwords:
|
||
headwords[reading].append(expression)
|
||
else:
|
||
headwords[reading] = [expression]
|
||
return headwords
|
||
|
||
def _find_expressions(self, soup):
|
||
self._delete_unused_nodes(soup)
|
||
text = soup.find("句表記").text
|
||
text = self._clean_expression(text)
|
||
alternatives = Expressions.expand_daijirin_alternatives(text)
|
||
expressions = []
|
||
for alt in alternatives:
|
||
for exp in Expressions.expand_abbreviation(alt):
|
||
expressions.append(exp)
|
||
return expressions
|
||
|
||
def _find_readings(self):
|
||
phrase_readings = load_phrase_readings(self.target)
|
||
text = phrase_readings[self.entry_id]
|
||
alternatives = Expressions.expand_daijirin_alternatives(text)
|
||
readings = []
|
||
for alt in alternatives:
|
||
for reading in Expressions.expand_abbreviation(alt):
|
||
readings.append(reading)
|
||
return readings
|