From 7ad72a6e4f6b952ebabd3b89d69a594c82e3cccb Mon Sep 17 00:00:00 2001 From: stephenmk Date: Fri, 5 May 2023 22:53:17 -0500 Subject: [PATCH] Add support for jitenon-kokugo --- bot/crawlers.py | 37 +++++ bot/entries/jitenon.py | 50 ++++++- bot/yomichan/export.py | 17 ++- bot/yomichan/glossary/jitenon.py | 237 +++++++++++++++++++++---------- bot/yomichan/terms/jitenon.py | 21 ++- data/yomichan_metadata.json | 9 ++ jitenbot.py | 2 + 7 files changed, 286 insertions(+), 87 deletions(-) diff --git a/bot/crawlers.py b/bot/crawlers.py index e520f76..7ba495c 100644 --- a/bot/crawlers.py +++ b/bot/crawlers.py @@ -4,11 +4,13 @@ from bs4 import BeautifulSoup import bot.scraper as Scraper +from bot.entries.jitenon import JitenonKokugoEntry from bot.entries.jitenon import JitenonKotowazaEntry from bot.entries.jitenon import JitenonYojiEntry from bot.entries.smk8 import Smk8Entry from bot.entries.daijirin2 import Daijirin2Entry +from bot.yomichan.export import JitenonKokugoExporter from bot.yomichan.export import JitenonKotowazaExporter from bot.yomichan.export import JitenonYojiExporter from bot.yomichan.export import Smk8Exporter @@ -48,6 +50,41 @@ class _Crawler(): return page_id +class JitenonKokugoCrawler(_Crawler): + def __init__(self, args): + super().__init__(args) + self._entry_class = JitenonKokugoEntry + self._yomi_exporter = JitenonKokugoExporter(args.target) + self._gojuon_url = "https://kokugo.jitenon.jp/cat/gojuonindex.php" + self._page_id_pattern = r"word/p([0-9]+)$" + + def collect_pages(self): + jitenon = Scraper.Jitenon() + gojuon_doc, _ = jitenon.scrape(self._gojuon_url) + gojuon_soup = BeautifulSoup(gojuon_doc, features="html.parser") + for gojuon_a in gojuon_soup.select(".kana_area a", href=True): + gojuon_href = gojuon_a['href'] + max_kana_page = 1 + current_kana_page = 1 + while current_kana_page <= max_kana_page: + kana_doc, _ = jitenon.scrape(f"{gojuon_href}&page={current_kana_page}") + current_kana_page += 1 + kana_soup = BeautifulSoup(kana_doc, features="html.parser") + page_total = kana_soup.find(class_="page_total").text + m = re.search(r"全([0-9]+)件", page_total) + if m: + max_kana_page = int(m.group(1)) + for kana_a in kana_soup.select(".word_box a", href=True): + page_link = kana_a['href'] + page_id = self._parse_page_id(page_link) + if page_id is None: + continue + _, page_path = jitenon.scrape(page_link) + self._page_map[page_id] = page_path + pages_len = len(self._page_map) + print(f"Finished scraping {pages_len} pages") + + class _JitenonCrawler(_Crawler): def __init__(self, args): super().__init__(args) diff --git a/bot/entries/jitenon.py b/bot/entries/jitenon.py index afff5b7..e1e17b4 100644 --- a/bot/entries/jitenon.py +++ b/bot/entries/jitenon.py @@ -7,8 +7,14 @@ import bot.expressions as Expressions class _JitenonEntry(Entry): + ID_TO_ENTRY = {} + def __init__(self, entry_id): super().__init__(entry_id) + if entry_id not in self.ID_TO_ENTRY: + self.ID_TO_ENTRY[entry_id] = self + else: + raise Exception(f"Duplicate entry ID: {entry_id}") self.modified_date = date(1970, 1, 1) self.attribution = "" for column in self._COLUMNS.values(): @@ -44,9 +50,9 @@ class _JitenonEntry(Entry): def _set_headwords(self): headwords = {} - for yomikata in self.__yomikatas(): + for yomikata in self._yomikatas(): headwords[yomikata] = [self.expression] - ikei_headwords = self.__ikei_headwords() + ikei_headwords = self._ikei_headwords() for reading, expressions in ikei_headwords.items(): if reading not in headwords: headwords[reading] = [] @@ -73,7 +79,7 @@ class _JitenonEntry(Entry): else: attr_value.append(colval) - def __yomikatas(self): + def _yomikatas(self): yomikata = self.yomikata m = re.search(r"^[ぁ-ヿ、]+$", yomikata) if m: @@ -94,7 +100,7 @@ class _JitenonEntry(Entry): print(f"Invalid 読み方 format: {self.yomikata}\n{self}\n") return [""] - def __ikei_headwords(self): + def _ikei_headwords(self): ikei_headwords = {} for val in self.ikei: m = re.search(r"^([^（]+)（([ぁ-ヿ、]+)）$", val) @@ -174,3 +180,39 @@ class JitenonKotowazaEntry(_JitenonEntry): for expressions in self._headwords.values(): Expressions.add_variant_kanji(expressions, self._variant_kanji) Expressions.add_fullwidth(expressions) + + +class JitenonKokugoEntry(_JitenonEntry): + _COLUMNS = { + "言葉": ["expression", ""], + "読み方": ["yomikata", ""], + "意味": ["imi", ""], + "例文": ["reibun", ""], + "別表記": ["betsuhyouki", ""], + "対義語": ["taigigo", ""], + "活用": ["katsuyou", ""], + "用例": ["yourei", ""], + "類語": ["ruigo", ""], + } + + def __init__(self, entry_id): + super().__init__(entry_id) + + def _set_headwords(self): + headwords = {} + for reading in self.yomikata.split("・"): + if reading not in headwords: + headwords[reading] = [] + for expression in self.expression.split("・"): + headwords[reading].append(expression) + if self.betsuhyouki.strip() != "": + for expression in self.betsuhyouki.split("・"): + headwords[reading].append(expression) + self._headwords = headwords + + def _set_variant_headwords(self): + for expressions in self._headwords.values(): + Expressions.add_variant_kanji(expressions, self._variant_kanji) + Expressions.add_fullwidth(expressions) + Expressions.remove_iteration_mark(expressions) + Expressions.add_iteration_mark(expressions) diff --git a/bot/yomichan/export.py b/bot/yomichan/export.py index 15b61c5..a2acf81 100644 --- a/bot/yomichan/export.py +++ b/bot/yomichan/export.py @@ -7,6 +7,7 @@ from platformdirs import user_documents_dir, user_cache_dir from bot.data import load_yomichan_metadata +from bot.yomichan.terms.jitenon import JitenonKokugoTerminator from bot.yomichan.terms.jitenon import JitenonYojiTerminator from bot.yomichan.terms.jitenon import JitenonKotowazaTerminator from bot.yomichan.terms.smk8 import Smk8Terminator @@ -20,8 +21,7 @@ class Exporter: self._terms_per_file = 2000 def export(self, entries, image_dir): - if image_dir is not None: - self.__init_build_image_dir(image_dir) + self.__init_build_image_dir(image_dir) meta = load_yomichan_metadata() index = meta[self._name]["index"] index["revision"] = self._get_revision(entries) @@ -42,10 +42,13 @@ class Exporter: return self._build_dir def __init_build_image_dir(self, image_dir): - print("Copying image files to build directory...") build_dir = self._get_build_dir() build_img_dir = os.path.join(build_dir, self._name) - shutil.copytree(image_dir, build_img_dir) + if image_dir is not None: + print("Copying image files to build directory...") + shutil.copytree(image_dir, build_img_dir) + else: + os.makedirs(build_img_dir) self._terminator.set_image_dir(build_img_dir) def __get_terms(self, entries): @@ -131,6 +134,12 @@ class JitenonExporter(Exporter): return attribution +class JitenonKokugoExporter(JitenonExporter): + def __init__(self, name): + super().__init__(name) + self._terminator = JitenonKokugoTerminator(name) + + class JitenonYojiExporter(JitenonExporter): def __init__(self, name): super().__init__(name) diff --git a/bot/yomichan/glossary/jitenon.py b/bot/yomichan/glossary/jitenon.py index ebee87c..6e3a192 100644 --- a/bot/yomichan/glossary/jitenon.py +++ b/bot/yomichan/glossary/jitenon.py @@ -1,93 +1,176 @@ import re +import os +from bs4 import BeautifulSoup +import bot.icons as Icons from bot.yomichan.glossary.gloss import make_gloss -def make_glossary(entry): - soup = entry.get_page_soup() - __replace_punctuation(soup) - __add_internal_links(soup) - __convert_paragraphs(soup) - __style_table_headers(soup) - __unwrap_table_body(soup) - __decompose_table_rows(soup, entry) - __insert_headword_line(soup, entry) - gloss = make_gloss(soup.body) - glossary = [gloss] - return glossary +class JitenonGlossary(): + def __init__(self): + self._id_pattern = None + self._expression_header = None + def _replace_punctuation(self, soup): + punctuation = { + "/": "／", + ",": "、", + } + for el in soup.find_all(string=True): + text = el.text + for old, new in punctuation.items(): + text = text.replace(old, new) + el.replace_with(text) -def __replace_punctuation(soup): - punctuation = { - "/": "／", - ",": "、", - } - for el in soup.find_all(string=True): - text = el.text - for old, new in punctuation.items(): - text = text.replace(old, new) - el.replace_with(text) + def _add_internal_links(self, soup, entry): + for el in soup.find_all("a"): + href = el.attrs["href"] + m = re.search(self._id_pattern, href) + if m is not None: + ref_entry_id = int(m.group(1)) + ref_entry = entry.ID_TO_ENTRY[ref_entry_id] + expression = ref_entry.get_first_expression() + el.attrs["href"] = f"?query={expression}&wildcards=off" + elif re.match(r"^(?:https?:|\?)[\w\W]*", href): + pass + else: + raise Exception(f"Invalid href format: {href}") + def _convert_paragraphs(self, soup): + for p in soup.find_all("p"): + p.name = "div" -def __add_internal_links(soup): - patterns = [ - r"^(.+)（[ぁ-ヿ、\s]+）$", - r"^(.+)（[ぁ-ヿ、\s]+（[ぁ-ヿ、\s]）[ぁ-ヿ、\s]+）$" - ] - for a in soup.find_all("a"): - for pattern in patterns: - m = re.search(pattern, a.text) - if m: - a['href'] = f"?query={m.group(1)}&wildcards=off" - break + def _style_table_headers(self, soup): + for th in soup.find_all("th"): + th['style'] = "vertical-align: middle; text-align: center;" + def _unwrap_table_body(self, soup): + if soup.find("tbody") is not None: + soup.tbody.unwrap() -def __convert_paragraphs(soup): - for p in soup.find_all("p"): - p.name = "span" - - -def __style_table_headers(soup): - for th in soup.find_all("th"): - th['style'] = "vertical-align: middle; text-align: center;" - - -def __unwrap_table_body(soup): - if soup.find("tbody") is not None: - soup.tbody.unwrap() - - -def __decompose_table_rows(soup, entry): - for tr in soup.find_all("tr"): - if tr.find("th") is None: - continue - elif tr.th.text in ["四字熟語", "言葉"]: - tr.decompose() - elif tr.th.text == "読み方": - if __do_display_yomikata_in_headword(entry): + def _decompose_table_rows(self, soup, entry): + for tr in soup.find_all("tr"): + if tr.find("th") is None: + continue + elif tr.th.text == self._expression_header: tr.decompose() - elif tr.th.text == "意味": - imi = tr.td - imi.name = "div" - soup.body.insert(0, imi) - tr.decompose() - if soup.find("tr") is None: - soup.table.decompose() + elif tr.th.text == "読み方": + if self._do_display_yomikata_in_headword(entry): + tr.decompose() + elif tr.th.text == "意味": + imi = tr.td + imi.name = "div" + soup.body.insert(0, imi) + tr.decompose() + if soup.find("tr") is None: + soup.table.decompose() + + def _insert_headword_line(self, soup, entry): + headword_line = soup.new_tag("span") + if self._do_display_yomikata_in_headword(entry): + headword_line.string = f"{entry.yomikata}【{entry.expression}】" + else: + headword_line.string = f"【{entry.expression}】" + soup.body.insert(0, headword_line) + + def _do_display_yomikata_in_headword(self, entry): + if not re.match(r"^[ぁ-ヿ、]+$", entry.yomikata): + return False + elif len(entry.yomikata) > 10: + return False + else: + return True -def __insert_headword_line(soup, entry): - headword_line = soup.new_tag("span") - if __do_display_yomikata_in_headword(entry): - headword_line.string = f"{entry.yomikata}【{entry.expression}】" - else: - headword_line.string = f"【{entry.expression}】" - soup.body.insert(0, headword_line) +class JitenonKokugoGlossary(JitenonGlossary): + def __init__(self): + super().__init__() + self._expression_header = "言葉" + self._id_pattern = r"kokugo.jitenon.jp/word/p([0-9]+)$" + + def make_glossary(self, entry, image_dir): + soup = entry.get_page_soup() + self._remove_antonym_list_item(soup) + self._replace_number_icons(soup, image_dir) + self._replace_punctuation(soup) + self._add_internal_links(soup, entry) + self._convert_paragraphs(soup) + self._style_table_headers(soup) + self._unwrap_table_body(soup) + self._decompose_table_rows(soup, entry) + self._insert_headword_line(soup, entry) + gloss = make_gloss(soup.body) + glossary = [gloss] + return glossary + + def _remove_antonym_list_item(self, soup): + for el in soup.find_all("li"): + if el.text == "対義語辞典": + el.decompose() + + def _replace_number_icons(self, soup, image_dir): + for el in soup.find_all("img"): + alt = el.attrs["alt"] + text = re.search(r"[０-９]+", alt).group(0) + filename = f"{text}-fill.svg" + path = os.path.join(image_dir, filename) + Icons.make_monochrome_fill_rectangle(path, text) + ratio = Icons.calculate_ratio(path) + img = BeautifulSoup("", "xml").img + img.attrs = { + "height": 1.0 if ratio > 1.0 else ratio, + "width": ratio if ratio > 1.0 else 1.0, + "sizeUnits": "em", + "collapsible": False, + "collapsed": False, + "background": False, + "appearance": "monochrome", + "title": alt, + "path": f"{os.path.basename(image_dir)}/{filename}", + } + el.name = "span" + el.append(img) + el.attrs["style"] = "margin-right: 0.25em;" + + def _do_display_yomikata_in_headword(self, entry): + return len(entry.yomikata) <= 10 -def __do_display_yomikata_in_headword(entry): - if not re.match(r"^[ぁ-ヿ、]+$", entry.yomikata): - return False - elif len(entry.yomikata) > 10: - return False - else: - return True +class JitenonYojiGlossary(JitenonGlossary): + def __init__(self): + super().__init__() + self._expression_header = "四字熟語" + self._id_pattern = r"yoji.jitenon.jp/yoji.?/([0-9]+)\.html$" + + def make_glossary(self, entry, image_dir): + soup = entry.get_page_soup() + self._replace_punctuation(soup) + self._add_internal_links(soup, entry) + self._convert_paragraphs(soup) + self._style_table_headers(soup) + self._unwrap_table_body(soup) + self._decompose_table_rows(soup, entry) + self._insert_headword_line(soup, entry) + gloss = make_gloss(soup.body) + glossary = [gloss] + return glossary + + +class JitenonKotowazaGlossary(JitenonGlossary): + def __init__(self): + super().__init__() + self._expression_header = "言葉" + self._id_pattern = r"kotowaza.jitenon.jp/kotowaza/([0-9]+)\.php$" + + def make_glossary(self, entry, image_dir): + soup = entry.get_page_soup() + self._replace_punctuation(soup) + self._add_internal_links(soup, entry) + self._convert_paragraphs(soup) + self._style_table_headers(soup) + self._unwrap_table_body(soup) + self._decompose_table_rows(soup, entry) + self._insert_headword_line(soup, entry) + gloss = make_gloss(soup.body) + glossary = [gloss] + return glossary diff --git a/bot/yomichan/terms/jitenon.py b/bot/yomichan/terms/jitenon.py index 75a3a5f..45f4d5b 100644 --- a/bot/yomichan/terms/jitenon.py +++ b/bot/yomichan/terms/jitenon.py @@ -1,6 +1,9 @@ from bot.yomichan.grammar import sudachi_rules from bot.yomichan.terms.terminator import Terminator -from bot.yomichan.glossary.jitenon import make_glossary + +from bot.yomichan.glossary.jitenon import JitenonKokugoGlossary +from bot.yomichan.glossary.jitenon import JitenonYojiGlossary +from bot.yomichan.glossary.jitenon import JitenonKotowazaGlossary class JitenonTerminator(Terminator): @@ -13,7 +16,7 @@ class JitenonTerminator(Terminator): def _glossary(self, entry): if entry.entry_id in self._glossary_cache: return self._glossary_cache[entry.entry_id] - glossary = make_glossary(entry) + glossary = self._glossary_maker.make_glossary(entry, self._image_dir) self._glossary_cache[entry.entry_id] = glossary return glossary @@ -27,9 +30,22 @@ class JitenonTerminator(Terminator): return [] +class JitenonKokugoTerminator(JitenonTerminator): + def __init__(self, name): + super().__init__(name) + self._glossary_maker = JitenonKokugoGlossary() + + def _inflection_rules(self, entry, expression): + return sudachi_rules(expression) + + def _term_tags(self, entry): + return "" + + class JitenonYojiTerminator(JitenonTerminator): def __init__(self, name): super().__init__(name) + self._glossary_maker = JitenonYojiGlossary() def _inflection_rules(self, entry, expression): return "" @@ -42,6 +58,7 @@ class JitenonYojiTerminator(JitenonTerminator): class JitenonKotowazaTerminator(JitenonTerminator): def __init__(self, name): super().__init__(name) + self._glossary_maker = JitenonKotowazaGlossary() def _inflection_rules(self, entry, expression): return sudachi_rules(expression) diff --git a/data/yomichan_metadata.json b/data/yomichan_metadata.json index c892015..3fa8cb4 100644 --- a/data/yomichan_metadata.json +++ b/data/yomichan_metadata.json @@ -1,4 +1,13 @@ { + "jitenon-kokugo": { + "index": { + "title": "国語辞典オンライン", + "sequenced": true, + "format": 3, + "url": "https://kokugo.jitenon.jp/" + }, + "tags": [] + }, "jitenon-yoji": { "index": { "title": "四字熟語辞典オンライン", diff --git a/jitenbot.py b/jitenbot.py index acd73a4..be42f5b 100644 --- a/jitenbot.py +++ b/jitenbot.py @@ -18,6 +18,7 @@ along with this program. If not, see . import os import argparse +from bot.crawlers import JitenonKokugoCrawler from bot.crawlers import JitenonYojiCrawler from bot.crawlers import JitenonKotowazaCrawler from bot.crawlers import Smk8Crawler @@ -59,6 +60,7 @@ def parse_args(targets): def main(): crawlers = { + "jitenon-kokugo": JitenonKokugoCrawler, "jitenon-yoji": JitenonYojiCrawler, "jitenon-kotowaza": JitenonKotowazaCrawler, "smk8": Smk8Crawler,