From d1d654ebac2d51e3841675faeb56480e440f622f Mon Sep 17 00:00:00 2001 From: Wolfgang Müller Date: Tue, 5 Mar 2024 18:08:09 +0100 Subject: Initial commit --- docs/_examples/example_scraper.json | 8 ++++++++ docs/_examples/example_scraper.py | 37 +++++++++++++++++++++++++++++++++++ docs/_examples/example_transformer.py | 21 ++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 docs/_examples/example_scraper.json create mode 100644 docs/_examples/example_scraper.py create mode 100644 docs/_examples/example_transformer.py (limited to 'docs/_examples') diff --git a/docs/_examples/example_scraper.json b/docs/_examples/example_scraper.json new file mode 100644 index 0000000..9efe126 --- /dev/null +++ b/docs/_examples/example_scraper.json @@ -0,0 +1,8 @@ +{ + "title": "This is a Title", + "tags": { + "artists": ["Alan Smithee", "Noah Ward"], + "characters": ["A", "B", "C"], + "misc": ["horror", "sci-fi"] + } +} diff --git a/docs/_examples/example_scraper.py b/docs/_examples/example_scraper.py new file mode 100644 index 0000000..d00c292 --- /dev/null +++ b/docs/_examples/example_scraper.py @@ -0,0 +1,37 @@ +import json + +from hircine.scraper import Scraper +from hircine.scraper.types import Artist, Character, Tag, Title +from hircine.scraper.utils import open_archive_file, parse_dict + + +class MyScraper(Scraper): + name = "Example scraper" + source = "example" + + def __init__(self, comic): + super().__init__(comic) + + self.data = self.load() + + if self.data: + self.is_available = True + + def load(self): + try: + with open_archive_file(self.comic.archive, "metadata.json") as jif: + return json.load(jif) + except Exception: + return {} + + def scrape(self): + parsers = { + "title": Title, + "tags": { + "artists": Artist, + "misc": Tag.from_string, + "characters": Character, + }, + } + + yield from parse_dict(parsers, self.data) diff --git a/docs/_examples/example_transformer.py b/docs/_examples/example_transformer.py new file mode 100644 index 0000000..6e443ae --- /dev/null +++ b/docs/_examples/example_transformer.py @@ -0,0 +1,21 @@ +from hircine.plugins import transformer +from hircine.scraper.types import Artist, Tag + + +@transformer +def transform(generator, info): + for item in generator: + # Ignore the "Drama" tag when scraping from mangadex + if info.source == "mangadex": + match item: + case Tag(tag="Drama"): + continue + + # convert all Artist names to lowercase + match item: + case Artist(name): + yield Artist(name.lower()) + continue + + # other items are not modified + yield item -- cgit v1.2.3-2-gb3c3