summaryrefslogtreecommitdiffstatshomepage
path: root/tests/plugins
diff options
context:
space:
mode:
Diffstat (limited to 'tests/plugins')
-rw-r--r--tests/plugins/scrapers/test_gallery_dl.py52
-rw-r--r--tests/plugins/scrapers/test_handlers.py594
2 files changed, 646 insertions, 0 deletions
diff --git a/tests/plugins/scrapers/test_gallery_dl.py b/tests/plugins/scrapers/test_gallery_dl.py
new file mode 100644
index 0000000..b4e7a4a
--- /dev/null
+++ b/tests/plugins/scrapers/test_gallery_dl.py
@@ -0,0 +1,52 @@
+import json
+import os
+from zipfile import ZipFile
+
+import pytest
+
+import hircine.plugins.scrapers.gallery_dl
+from hircine.plugins.scrapers.gallery_dl import GalleryDLScraper
+from hircine.scraper.types import Title
+
+
+class MockHandler:
+ source = "mock"
+
+ def scrape(self, data):
+ yield Title(data["title"])
+
+
+@pytest.fixture
+def archive_file(tmpdir):
+ file = os.path.join(tmpdir, "archive.zip")
+
+ with ZipFile(file, "x") as ziph:
+ ziph.writestr("info.json", json.dumps({"category": "mock", "title": "test"}))
+
+ yield file
+
+
+def test_does_scrape(monkeypatch, archive_file, gen_comic):
+ comic = next(gen_comic)
+ comic.archive.path = archive_file
+
+ monkeypatch.setattr(
+ hircine.plugins.scrapers.gallery_dl, "HANDLERS", {"mock": MockHandler}
+ )
+
+ scraper = GalleryDLScraper(comic)
+
+ assert scraper.is_available
+ assert scraper.source == MockHandler.source
+ assert scraper.name == f"gallery-dl info.json ({MockHandler.source})"
+ assert set(scraper.collect()) == set([Title(value="test")])
+
+
+def test_does_not_scrape_on_error(tmpdir, monkeypatch, gen_comic):
+ comic = next(gen_comic)
+ comic.archive.path = os.path.join(tmpdir, "nonexistent.zip")
+
+ scraper = GalleryDLScraper(comic)
+
+ assert scraper.data == {}
+ assert not scraper.is_available
diff --git a/tests/plugins/scrapers/test_handlers.py b/tests/plugins/scrapers/test_handlers.py
new file mode 100644
index 0000000..e9f5d0e
--- /dev/null
+++ b/tests/plugins/scrapers/test_handlers.py
@@ -0,0 +1,594 @@
+import json
+from datetime import date
+
+import pytest
+
+import hircine.enums as enums
+from hircine.plugins.scrapers.handlers.dynastyscans import DynastyScansHandler
+from hircine.plugins.scrapers.handlers.e621 import E621Handler
+from hircine.plugins.scrapers.handlers.exhentai import (
+ ExHentaiHandler,
+)
+from hircine.plugins.scrapers.handlers.exhentai import (
+ sanitize as exhentai_sanitize,
+)
+from hircine.plugins.scrapers.handlers.mangadex import MangadexHandler
+from hircine.scraper import Scraper
+from hircine.scraper.types import (
+ URL,
+ Artist,
+ Category,
+ Censorship,
+ Character,
+ Circle,
+ Date,
+ Direction,
+ Language,
+ OriginalTitle,
+ Rating,
+ Tag,
+ Title,
+ World,
+)
+
+
+class Scraper(Scraper):
+ def __init__(self, handler, json):
+ self.handler = handler
+ self.json = json
+ super().__init__(None)
+
+ def scrape(self):
+ yield from self.handler.scrape(json.loads(self.json))
+
+
+def test_dynastyscans():
+ scraper = Scraper(
+ DynastyScansHandler(),
+ """
+ {
+ "manga": "Hoshiiro GirlDrop Comic Anthology",
+ "chapter": 1,
+ "chapter_minor": "",
+ "title": "Hop, Step, Drop!",
+ "author": "Fujisawa Kamiya",
+ "group": "Cyan Steam (Stan Miller)",
+ "date": "2018-02-05 00:00:00",
+ "lang": "en",
+ "language": "English",
+ "count": 15,
+ "category": "dynastyscans",
+ "subcategory": "manga"
+ }
+ """,
+ )
+
+ assert set(scraper.collect()) == set(
+ [
+ Artist(name="Fujisawa Kamiya"),
+ Circle(name="Cyan Steam (Stan Miller)"),
+ Date(value=date(2018, 2, 5)),
+ Language(value=enums.Language.EN),
+ Title(value="Hoshiiro GirlDrop Comic Anthology Ch. 1: Hop, Step, Drop!"),
+ ]
+ )
+
+
+def test_mangadex():
+ scraper = Scraper(
+ MangadexHandler(),
+ """
+ {
+ "manga": "Shimeji Simulation",
+ "manga_id": "28b5d037-175d-4119-96f8-e860e408ebe9",
+ "title": "Danchi",
+ "volume": 1,
+ "chapter": 8,
+ "chapter_minor": "",
+ "chapter_id": "2a115ccb-de52-4b84-9166-cebd152d9396",
+ "date": "2019-09-22 04:19:15",
+ "lang": "en",
+ "language": "English",
+ "count": 12,
+ "artist": [
+ "Tsukumizu"
+ ],
+ "author": [
+ "Tsukumizu"
+ ],
+ "group": [
+ "Orchesc/a/ns"
+ ],
+ "status": "completed",
+ "tags": [
+ "Sci-Fi",
+ "Comedy",
+ "Girls' Love",
+ "4-Koma",
+ "Philosophical",
+ "School Life",
+ "Slice of Life"
+ ],
+ "category": "mangadex",
+ "subcategory": "chapter"
+ }
+ """,
+ )
+
+ assert set(scraper.collect()) == set(
+ [
+ Artist(name="Tsukumizu"),
+ Circle(name="Orchesc/a/ns"),
+ Date(value=date(2019, 9, 22)),
+ Language(value=enums.Language.EN),
+ Tag(namespace="none", tag="4-Koma"),
+ Tag(namespace="none", tag="Comedy"),
+ Tag(namespace="none", tag="Girls' Love"),
+ Tag(namespace="none", tag="Philosophical"),
+ Tag(namespace="none", tag="School Life"),
+ Tag(namespace="none", tag="Sci-Fi"),
+ Tag(namespace="none", tag="Slice of Life"),
+ Title(value="Shimeji Simulation Vol. 1, Ch. 8: Danchi"),
+ URL("https://mangadex.org/chapter/2a115ccb-de52-4b84-9166-cebd152d9396"),
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ "data, title",
+ [
+ ({"volume": 1, "chapter": 8}, "Manga Vol. 1, Ch. 8: Title"),
+ ({"volume": 0, "chapter": 1}, "Manga Ch. 1: Title"),
+ ({"volume": 0, "chapter": 0}, "Manga: Title"),
+ ],
+ ids=[
+ "volume and chapter",
+ "chapter only",
+ "none",
+ ],
+)
+def test_mangadex_handles_volume_and_chapter(data, title):
+ common = {"manga": "Manga", "title": "Title"}
+ scraper = Scraper(MangadexHandler(), json.dumps(common | data))
+
+ assert list(scraper.collect()) == [Title(value=title)]
+
+
+def test_e621_pool():
+ scraper = Scraper(
+ E621Handler(),
+ """
+ {
+ "id": 2968472,
+ "created_at": "2021-10-10T04:13:53.286-04:00",
+ "updated_at": "2024-11-02T08:58:06.724-04:00",
+ "file": {
+ "width": 800,
+ "height": 800,
+ "ext": "jpg",
+ "size": 530984,
+ "md5": "1ec7e397bb22c1454ab1986fd3f3edc5",
+ "url": "https://static1.e621.net/data/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg"
+ },
+ "preview": {
+ "width": 150,
+ "height": 150,
+ "url": "https://static1.e621.net/data/preview/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg"
+ },
+ "sample": {
+ "has": false,
+ "height": 800,
+ "width": 800,
+ "url": "https://static1.e621.net/data/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg",
+ "alternates": {}
+ },
+ "score": {
+ "up": 202,
+ "down": -1,
+ "total": 201
+ },
+ "tags": {
+ "general": [
+ "beak"
+ ],
+ "artist": [
+ "falseknees"
+ ],
+ "copyright": [],
+ "character": [],
+ "species": [
+ "bird"
+ ],
+ "invalid": [],
+ "meta": [
+ "comic",
+ "english_text"
+ ],
+ "lore": [
+ "parent_(lore)",
+ "parent_and_child_(lore)"
+ ]
+ },
+ "locked_tags": [],
+ "change_seq": 60808337,
+ "flags": {
+ "pending": false,
+ "flagged": false,
+ "note_locked": false,
+ "status_locked": false,
+ "rating_locked": false,
+ "deleted": false
+ },
+ "rating": "s",
+ "fav_count": 194,
+ "sources": [
+ "https://twitter.com/FalseKnees/status/1324869853627478022"
+ ],
+ "pools": [
+ 25779
+ ],
+ "relationships": {
+ "parent_id": null,
+ "has_children": false,
+ "has_active_children": false,
+ "children": []
+ },
+ "approver_id": 171673,
+ "uploader_id": 178921,
+ "description": "",
+ "comment_count": 1,
+ "is_favorited": false,
+ "has_notes": false,
+ "duration": null,
+ "num": 1,
+ "filename": "1ec7e397bb22c1454ab1986fd3f3edc5",
+ "extension": "jpg",
+ "date": "2021-10-10 08:13:53",
+ "pool": {
+ "id": 25779,
+ "name": "Kids say the darnedest shit - falseknees",
+ "created_at": "2021-10-10T04:17:07.006-04:00",
+ "updated_at": "2021-10-10T04:17:07.006-04:00",
+ "creator_id": 178921,
+ "description": "The terror of every parent.",
+ "is_active": true,
+ "category": "series",
+ "creator_name": "OneMoreAnonymous",
+ "post_count": 4
+ },
+ "category": "e621",
+ "subcategory": "pool"
+ }
+ """,
+ )
+
+ assert set(scraper.collect()) == set(
+ [
+ Artist(name="falseknees"),
+ Category(value=enums.Category.COMIC),
+ Censorship(value=enums.Censorship.NONE),
+ Date(value=date(2021, 10, 10)),
+ Language(value=enums.Language.EN),
+ Rating(value=enums.Rating.SAFE),
+ Tag(namespace="none", tag="beak"),
+ Tag(namespace="none", tag="bird"),
+ Title(value="Kids say the darnedest shit - falseknees"),
+ URL("https://e621.net/pools/25779"),
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ "data, censorship",
+ [
+ ({"tags": {"meta": ["censor_bar"]}}, enums.Censorship.BAR),
+ ({"tags": {"meta": ["mosaic_censorship"]}}, enums.Censorship.MOSAIC),
+ ({"tags": {"meta": ["uncensored"]}}, enums.Censorship.NONE),
+ ({"tags": {"meta": []}}, enums.Censorship.NONE),
+ ],
+ ids=[
+ "bars",
+ "mosaic",
+ "uncensored",
+ "uncensored (implied)",
+ ],
+)
+def test_e621_handles_censorship(data, censorship):
+ common = {"subcategory": "pool"}
+ scraper = Scraper(E621Handler(), json.dumps(common | data))
+
+ assert set(scraper.collect()) == set([Censorship(value=censorship)])
+
+
+def test_exhentai_explicit():
+ scraper = Scraper(
+ ExHentaiHandler(),
+ """
+ {
+ "gid": 2771624,
+ "token": "43108ee23b",
+ "thumb": "https://s.exhentai.org/t/12/80/1280a064a2ab3d70b9feb56bd0c55dbfc3ab6a39-309830-950-1351-jpg_250.jpg",
+ "title": "[NAGABE] Smell ch.01 - ch.06",
+ "title_jpn": "SMELL",
+ "eh_category": "Doujinshi",
+ "uploader": "randaldog",
+ "date": "2023-12-19 23:50:00",
+ "parent": "https://exhentai.org/g/2736803/b191bfed72/",
+ "expunged": false,
+ "language": "English",
+ "filesize": 74469868,
+ "filecount": "170",
+ "favorites": "751",
+ "rating": "4.83",
+ "torrentcount": "0",
+ "lang": "en",
+ "tags": [
+ "language:english",
+ "language:translated",
+ "parody:original",
+ "artist:nagabe",
+ "male:dog boy",
+ "male:furry",
+ "male:males only",
+ "male:smell",
+ "male:yaoi",
+ "other:story arc"
+ ],
+ "category": "exhentai",
+ "subcategory": "gallery"
+ }
+ """,
+ )
+
+ assert set(scraper.collect()) == set(
+ [
+ Artist(name="nagabe"),
+ Category(value=enums.Category.DOUJINSHI),
+ Censorship(value=enums.Censorship.BAR),
+ Date(value=date(2023, 12, 19)),
+ Direction(value=enums.Direction.RIGHT_TO_LEFT),
+ Language(value=enums.Language.EN),
+ OriginalTitle(value="SMELL"),
+ Rating(value=enums.Rating.EXPLICIT),
+ Tag(namespace="male", tag="dog boy"),
+ Tag(namespace="male", tag="furry"),
+ Tag(namespace="male", tag="males only"),
+ Tag(namespace="male", tag="smell"),
+ Tag(namespace="male", tag="yaoi"),
+ Tag(namespace="other", tag="story arc"),
+ Title(value="Smell ch.01 - ch.06"),
+ URL("https://exhentai.org/g/2771624/43108ee23b"),
+ World(name="original"),
+ ]
+ )
+
+
+def test_exhentai_non_h():
+ scraper = Scraper(
+ ExHentaiHandler(),
+ """
+ {
+ "gid": 1025913,
+ "token": "fdaabef1a2",
+ "thumb": "https://s.exhentai.org/t/51/17/5117cde63cc14436c5ad7f2dd06abb52c86aff65-23642001-2866-4047-png_250.jpg",
+ "title": "(C91) [Animachine (Shimahara)] Iya na Kao Sarenagara Opantsu Misete Moraitai Manga | A manga about girl showing you her panties while making a disgusted face [English] [葛の寺]",
+ "title_jpn": "(C91) [アニマルマシーン (40原)] 嫌な顔されながらおパンツ見せてもらいたい漫画 [英訳]",
+ "eh_category": "Non-H",
+ "uploader": "葛の寺",
+ "date": "2017-02-04 04:25:00",
+ "parent": "https://exhentai.org/g/1025875/cfe6adccb8/",
+ "expunged": false,
+ "language": "English",
+ "filesize": 0,
+ "filecount": "23",
+ "favorites": "1088",
+ "rating": "4.74",
+ "torrentcount": "1",
+ "lang": "en",
+ "tags": [
+ "language:english",
+ "language:translated",
+ "parody:iya na kao sare nagara opantsu misete moraitai",
+ "group:animachine",
+ "artist:shimahara",
+ "female:femdom",
+ "female:schoolgirl uniform",
+ "other:full color"
+ ],
+ "category": "exhentai",
+ "subcategory": "gallery"
+ }
+ """, # noqa: E501
+ )
+
+ assert set(scraper.collect()) == set(
+ [
+ Artist(name="shimahara"),
+ Date(value=date(2017, 2, 4)),
+ Language(value=enums.Language.EN),
+ OriginalTitle(value="嫌な顔されながらおパンツ見せてもらいたい漫画"),
+ Rating(value=enums.Rating.QUESTIONABLE),
+ World(name="iya na kao sare nagara opantsu misete moraitai"),
+ Circle(name="animachine"),
+ Tag(namespace="female", tag="femdom"),
+ Tag(namespace="female", tag="schoolgirl uniform"),
+ Tag(namespace="other", tag="full color"),
+ Title(
+ value="A manga about girl showing you her panties while making a disgusted face" # noqa: E501
+ ),
+ URL("https://exhentai.org/g/1025913/fdaabef1a2"),
+ ]
+ )
+
+
+@pytest.mark.parametrize(
+ "text, sanitized",
+ [
+ ("(foo) Title", "Title"),
+ ("[foo] {bar} =baz= Title", "Title"),
+ ("Foreign Title | Localized Title", "Localized Title"),
+ ],
+ ids=[
+ "parens at beginning",
+ "bracket-likes",
+ "split titles",
+ ],
+)
+def test_exhentai_sanitizes(text, sanitized):
+ assert exhentai_sanitize(text, split=True) == sanitized
+
+
+@pytest.mark.parametrize(
+ "data, expect",
+ [
+ (
+ {"category": "doujinshi"},
+ set(
+ [
+ Category(value=enums.Category.DOUJINSHI),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ Direction(value=enums.Direction.RIGHT_TO_LEFT),
+ ]
+ ),
+ ),
+ (
+ {"eh_category": "doujinshi"},
+ set(
+ [
+ Category(value=enums.Category.DOUJINSHI),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ Direction(value=enums.Direction.RIGHT_TO_LEFT),
+ ]
+ ),
+ ),
+ (
+ {"category": "manga"},
+ set(
+ [
+ Category(value=enums.Category.MANGA),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ Direction(value=enums.Direction.RIGHT_TO_LEFT),
+ ]
+ ),
+ ),
+ (
+ {"category": "western"},
+ set(
+ [
+ Censorship(value=enums.Censorship.NONE),
+ Rating(value=enums.Rating.EXPLICIT),
+ ]
+ ),
+ ),
+ (
+ {"category": "artist cg"},
+ set(
+ [
+ Category(value=enums.Category.COMIC),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ ]
+ ),
+ ),
+ (
+ {"category": "game cg"},
+ set(
+ [
+ Category(value=enums.Category.GAME_CG),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ ]
+ ),
+ ),
+ (
+ {"category": "image set"},
+ set(
+ [
+ Category(value=enums.Category.IMAGE_SET),
+ Censorship(value=enums.Censorship.BAR),
+ Rating(value=enums.Rating.EXPLICIT),
+ ]
+ ),
+ ),
+ (
+ {"category": "non-h"},
+ set(
+ [
+ Rating(value=enums.Rating.QUESTIONABLE),
+ ]
+ ),
+ ),
+ (
+ {"category": "western", "tags": ["other:western non-h"]},
+ set(
+ [
+ Rating(value=enums.Rating.QUESTIONABLE),
+ ]
+ ),
+ ),
+ ],
+ ids=[
+ "category from category field",
+ "category from eh_category field",
+ "manga category",
+ "western category",
+ "artist cg category",
+ "game cg category",
+ "image set category",
+ "non-h category",
+ "western non-h tag",
+ ],
+)
+def test_exhentai_parses(data, expect):
+ scraper = Scraper(ExHentaiHandler(), json.dumps(data | {"gid": 1, "token": 1}))
+
+ expect.add(URL(value="https://exhentai.org/g/1/1"))
+
+ assert set(scraper.collect()) == expect
+
+
+@pytest.mark.parametrize(
+ "tag, parsed",
+ [
+ ("parody:foo", World(name="foo")),
+ ("artist:foo", Artist(name="foo")),
+ ("character:foo", Character(name="foo")),
+ ("group:foo", Circle(name="foo")),
+ ("other:artbook", Category(value=enums.Category.ARTBOOK)),
+ ("other:non-h imageset", Category(value=enums.Category.IMAGE_SET)),
+ ("other:western imageset", Category(value=enums.Category.IMAGE_SET)),
+ ("other:comic", Category(value=enums.Category.COMIC)),
+ ("other:variant set", Category(value=enums.Category.VARIANT_SET)),
+ ("other:webtoon", Category(value=enums.Category.WEBTOON)),
+ ("other:full censorship", Censorship(value=enums.Censorship.FULL)),
+ ("other:mosaic censorship", Censorship(value=enums.Censorship.MOSAIC)),
+ ("other:uncensored", Censorship(value=enums.Censorship.NONE)),
+ ("generic", Tag(namespace=None, tag="generic")),
+ ],
+ ids=[
+ "parody",
+ "group",
+ "artist",
+ "character",
+ "other:artbook",
+ "other:image set",
+ "other:western image set",
+ "other:comic",
+ "other:variant set",
+ "other:webtoon",
+ "other:full censorship",
+ "other:mosaic censorship",
+ "other:uncensored",
+ "generic",
+ ],
+)
+def test_exhentai_parses_tags(tag, parsed):
+ scraper = Scraper(
+ ExHentaiHandler(), json.dumps({"tags": [tag], "gid": 1, "token": 1})
+ )
+ expect = set([URL(value="https://exhentai.org/g/1/1"), parsed])
+
+ assert set(scraper.collect()) > expect
powered by cgit and git | oriole.systems | how to verify signatures