diff options
Diffstat (limited to '')
29 files changed, 1427 insertions, 71 deletions
diff --git a/tests/api/test_archive.py b/tests/api/test_archive.py index 0ef3425..6e6d0b7 100644 --- a/tests/api/test_archive.py +++ b/tests/api/test_archive.py @@ -2,13 +2,14 @@ import os from datetime import datetime as dt from pathlib import Path +import pytest +from conftest import DB, Response +from sqlalchemy import select + import hircine.config import hircine.db as database import hircine.thumbnailer as thumb -import pytest -from conftest import DB, Response from hircine.db.models import Archive, Comic, Image, Page -from sqlalchemy import select @pytest.fixture diff --git a/tests/api/test_artist.py b/tests/api/test_artist.py index 8cb2f1a..fa58012 100644 --- a/tests/api/test_artist.py +++ b/tests/api/test_artist.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import Artist diff --git a/tests/api/test_character.py b/tests/api/test_character.py index 567d2a4..3737d49 100644 --- a/tests/api/test_character.py +++ b/tests/api/test_character.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import Character diff --git a/tests/api/test_circle.py b/tests/api/test_circle.py index a03ba89..bea46d7 100644 --- a/tests/api/test_circle.py +++ b/tests/api/test_circle.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import Circle diff --git a/tests/api/test_comic.py b/tests/api/test_comic.py index d3fa51e..dcc5822 100644 --- a/tests/api/test_comic.py +++ b/tests/api/test_comic.py @@ -3,6 +3,7 @@ from datetime import datetime as dt import pytest from conftest import DB, Response + from hircine.db.models import ( Artist, Circle, @@ -1096,7 +1097,7 @@ async def test_upsert_comic_tags_uses_existing(upsert_comics, empty_comic): @pytest.mark.parametrize( - "key,list", + "key,items", [ ("artists", ["arty", "farty"]), ("tags", ["alien:medium", "human:tiny"]), @@ -1115,11 +1116,11 @@ async def test_upsert_comic_tags_uses_existing(upsert_comics, empty_comic): ], ) @pytest.mark.anyio -async def test_upsert_comic_creates(upsert_comics, empty_comic, key, list): +async def test_upsert_comic_creates(upsert_comics, empty_comic, key, items): original_comic = await DB.add(empty_comic) input = { - key: {"names": list, "options": {"onMissing": "CREATE"}}, + key: {"names": items, "options": {"onMissing": "CREATE"}}, } response = Response(await upsert_comics(original_comic.id, input)) response.assert_is("UpsertSuccess") @@ -1127,7 +1128,7 @@ async def test_upsert_comic_creates(upsert_comics, empty_comic, key, list): comic = await DB.get(Comic, original_comic.id, full=True) assert comic is not None - assert set(list) == set([o.name for o in getattr(comic, key)]) + assert set(items) == set([o.name for o in getattr(comic, key)]) @pytest.mark.anyio @@ -1184,7 +1185,9 @@ async def test_upsert_comic_fails_creating_invalid_tag(upsert_comics, gen_comic, response = Response(await upsert_comics(comic.id, input)) response.assert_is("InvalidParameterError") assert response.parameter == "name" - msg = "Invalid parameter 'name': ComicTag name must be specified as <namespace>:<tag>" # noqa: E501 + msg = ( + "Invalid parameter 'name': ComicTag name must be specified as <namespace>:<tag>" + ) assert response.message == msg diff --git a/tests/api/test_comic_tag.py b/tests/api/test_comic_tag.py index f536b79..d0878e2 100644 --- a/tests/api/test_comic_tag.py +++ b/tests/api/test_comic_tag.py @@ -2,6 +2,7 @@ from functools import partial import pytest from conftest import DB, Response + from hircine.db.models import Namespace, Tag diff --git a/tests/api/test_db.py b/tests/api/test_db.py index f53b90f..b030035 100644 --- a/tests/api/test_db.py +++ b/tests/api/test_db.py @@ -1,10 +1,16 @@ from datetime import datetime, timedelta, timezone +import pytest +from conftest import DB +from sqlalchemy.exc import StatementError +from sqlalchemy.orm import ( + Mapped, + mapped_column, +) + import hircine.db as database import hircine.db.models as models import hircine.db.ops as ops -import pytest -from conftest import DB from hircine.db.models import ( Artist, Base, @@ -16,11 +22,6 @@ from hircine.db.models import ( Tag, TagNamespaces, ) -from sqlalchemy.exc import StatementError -from sqlalchemy.orm import ( - Mapped, - mapped_column, -) class Date(MixinID, Base): @@ -66,8 +67,8 @@ async def test_models_retained_when_clearing_association( comic = await DB.add(comic) async with database.session() as s: - object = await s.get(Comic, comic.id) - setattr(object, key, []) + obj = await s.get(Comic, comic.id) + setattr(obj, key, []) await s.commit() assert await DB.get(assoccls, (comic.id, model.id)) is None @@ -86,8 +87,8 @@ async def test_models_retained_when_clearing_comictag(empty_comic): await DB.add(ct) async with database.session() as s: - object = await s.get(Comic, comic.id) - object.tags = [] + obj = await s.get(Comic, comic.id) + obj.tags = [] await s.commit() assert await DB.get(ComicTag, (comic.id, ct.namespace_id, ct.tag_id)) is None diff --git a/tests/api/test_filter.py b/tests/api/test_filter.py index 67a953f..6eb2934 100644 --- a/tests/api/test_filter.py +++ b/tests/api/test_filter.py @@ -1,5 +1,6 @@ import pytest from conftest import DB, Response + from hircine.db.models import Namespace, Tag @@ -420,51 +421,59 @@ async def test_field_presence(query_comic_filter, gen_comic, empty_comic, filter "filter,ids", [ ( - {"include": {"artists": {"empty": True}}}, + {"include": {"artists": {"count": {"value": 0}}}}, [100], ), ( - {"include": {"artists": {"empty": False}}}, - [1, 2], + {"include": {"artists": {"count": {"value": 0, "operator": "EQUAL"}}}}, + [100], ), ( - {"exclude": {"artists": {"empty": True}}}, - [1, 2], + { + "include": { + "artists": {"count": {"value": 1, "operator": "GREATER_THAN"}} + } + }, + [1], ), ( - {"exclude": {"artists": {"empty": False}}}, - [100], + {"include": {"artists": {"count": {"value": 3, "operator": "LOWER_THAN"}}}}, + [1, 2, 100], ), ( - {"include": {"tags": {"empty": True}}}, - [100], + {"exclude": {"artists": {"count": {"value": 0}}}}, + [1, 2], ), ( - {"include": {"tags": {"empty": False}}}, + {"exclude": {"artists": {"count": {"value": 0, "operator": "EQUAL"}}}}, [1, 2], ), ( - {"exclude": {"tags": {"empty": True}}}, - [1, 2], + { + "exclude": { + "artists": {"count": {"value": 1, "operator": "GREATER_THAN"}} + } + }, + [2, 100], ), ( - {"exclude": {"tags": {"empty": False}}}, - [100], + {"exclude": {"artists": {"count": {"value": 3, "operator": "LOWER_THAN"}}}}, + [], ), ], ids=[ - "includes artist empty", - "includes artist not empty", - "excludes artist empty", - "excludes artist not empty", - "includes tags empty", - "includes tags not empty", - "excludes tags empty", - "excludes tags not empty", + "include equal (default)", + "include equal (explicit)", + "include greater than", + "include lower than", + "exclude equal (default)", + "exclude equal (explicit)", + "exclude greater than", + "exclude lower than", ], ) @pytest.mark.anyio -async def test_assoc_presence(query_comic_filter, gen_comic, empty_comic, filter, ids): +async def test_assoc_counts(query_comic_filter, gen_comic, empty_comic, filter, ids): await DB.add(next(gen_comic)) await DB.add(next(gen_comic)) await DB.add(empty_comic) @@ -519,3 +528,48 @@ async def test_tag_assoc_filter(query_tag_filter, gen_namespace, gen_tag, filter response.assert_is("TagFilterResult") assert id_list(response.edges) == ids + + +@pytest.mark.parametrize( + "filter,expect", + [ + ({"include": {"comics": {"count": {"value": 1}}}}, [2, 3]), + ({"include": {"comics": {"count": {"value": 2, "operator": "EQUAL"}}}}, [1, 4]), + ( + { + "include": { + "comics": {"count": {"value": 3, "operator": "GREATER_THAN"}} + } + }, + [], + ), + ( + {"include": {"comics": {"count": {"value": 2, "operator": "LOWER_THAN"}}}}, + [2, 3], + ), + ( + {"exclude": {"comics": {"count": {"value": 1}}}}, + [1, 4], + ), + ( + {"exclude": {"comics": {"count": {"value": 1, "operator": "LOWER_THAN"}}}}, + [1, 2, 3, 4], + ), + ], + ids=[ + "include equal (default)", + "include equal (explicit)", + "include greater than", + "include lower than", + "exclude equal (default)", + "exclude lower than", + ], +) +@pytest.mark.anyio +async def test_count_filter(query_string_filter, gen_comic, filter, expect): + await DB.add_all(*gen_comic) + + response = Response(await query_string_filter(filter)) + response.assert_is("ArtistFilterResult") + + assert id_list(response.edges) == expect diff --git a/tests/api/test_image.py b/tests/api/test_image.py index c8c26b3..e0e9251 100644 --- a/tests/api/test_image.py +++ b/tests/api/test_image.py @@ -1,5 +1,6 @@ import pytest from conftest import DB + from hircine.api.types import Image diff --git a/tests/api/test_namespace.py b/tests/api/test_namespace.py index 450075b..2ffc118 100644 --- a/tests/api/test_namespace.py +++ b/tests/api/test_namespace.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import Namespace diff --git a/tests/api/test_page.py b/tests/api/test_page.py index debd69a..cb06e3e 100644 --- a/tests/api/test_page.py +++ b/tests/api/test_page.py @@ -2,6 +2,7 @@ from datetime import datetime, timezone import pytest from conftest import DB + from hircine.api.types import Page from hircine.db.models import Archive diff --git a/tests/api/test_scraper_api.py b/tests/api/test_scraper_api.py index 1edd74f..b917e39 100644 --- a/tests/api/test_scraper_api.py +++ b/tests/api/test_scraper_api.py @@ -1,8 +1,9 @@ +import pytest +from conftest import DB, Response + import hircine.enums as enums import hircine.plugins import hircine.scraper.types as scraped -import pytest -from conftest import DB, Response from hircine.scraper import ScrapeError, Scraper, ScrapeWarning diff --git a/tests/api/test_sort.py b/tests/api/test_sort.py index b3c8562..02a7ec3 100644 --- a/tests/api/test_sort.py +++ b/tests/api/test_sort.py @@ -1,6 +1,7 @@ import pytest from conftest import DB, Response -from hircine.db.models import Namespace + +from hircine.db.models import Namespace, Tag @pytest.fixture @@ -22,6 +23,24 @@ def query_comic_sort(execute_sort): @pytest.fixture +def query_artist_sort(execute_sort): + query = """ + query artists($sort: ArtistSortInput) { + artists(sort: $sort) { + __typename + count + edges { + id + name + } + } + } + """ + + return execute_sort(query) + + +@pytest.fixture def query_namespace_sort(execute_sort): query = """ query namespaces($sort: NamespaceSortInput) { @@ -87,6 +106,31 @@ async def test_query_comics_sort_tag_count(gen_comic, query_comic_sort, sort, re assert ids == [edge["id"] for edge in response.edges] +@pytest.mark.parametrize( + "sort,reverse,expect", + [ + ({"on": "COMIC_COUNT"}, False, [2, 3, 1, 4]), + ({"on": "COMIC_COUNT", "direction": "DESCENDING"}, True, [1, 4, 2, 3]), + ({"on": "COMIC_COUNT", "direction": "ASCENDING"}, False, [2, 3, 1, 4]), + ], + ids=[ + "ascending (default)", + "descending", + "ascending", + ], +) +@pytest.mark.anyio +async def test_query_artists_sort_comic_count( + gen_comic, query_artist_sort, sort, reverse, expect +): + await DB.add_all(*gen_comic) + + response = Response(await query_artist_sort(sort)) + response.assert_is("ArtistFilterResult") + + assert expect == [edge["id"] for edge in response.edges] + + @pytest.mark.anyio async def test_query_comics_sort_random(gen_comic, query_comic_sort): comics = await DB.add_all(*gen_comic) @@ -134,4 +178,35 @@ async def test_query_namespace_sort_sort_name(query_namespace_sort): response = Response(await query_namespace_sort({"on": "SORT_NAME"})) response.assert_is("NamespaceFilterResult") - assert ["two", "one"] == [edge["name"] for edge in response.edges] + assert [edge["name"] for edge in response.edges] == ["two", "one"] + + +@pytest.mark.parametrize( + "sort,reverse,expect", + [ + ({"on": "TAG_COUNT"}, False, [2, 1]), + ({"on": "TAG_COUNT", "direction": "DESCENDING"}, True, [1, 2]), + ({"on": "TAG_COUNT", "direction": "ASCENDING"}, False, [2, 1]), + ], + ids=[ + "ascending (default)", + "descending", + "ascending", + ], +) +@pytest.mark.anyio +async def test_query_namespace_sort_tag_count( + gen_comic, query_namespace_sort, sort, reverse, expect +): + namespace_foo = Namespace(id=1, name="foo") + namespace_bar = Namespace(id=2, name="bar") + + tag_foo = Tag(id=1, name="foo", namespaces=[namespace_foo]) + tag_bar = Tag(id=2, name="bar", namespaces=[namespace_foo, namespace_bar]) + + await DB.add_all(tag_foo, tag_bar) + + response = Response(await query_namespace_sort(sort)) + response.assert_is("NamespaceFilterResult") + + assert expect == [edge["id"] for edge in response.edges] diff --git a/tests/api/test_statistics.py b/tests/api/test_statistics.py new file mode 100644 index 0000000..98c8dc7 --- /dev/null +++ b/tests/api/test_statistics.py @@ -0,0 +1,106 @@ +import pytest +from conftest import DB, Response + +import hircine.plugins +from hircine.db.models import ( + Artist, + Character, + Circle, + Page, + Tag, + World, +) +from hircine.scraper import Scraper + +totals_fragment = """ + fragment Totals on Statistics { + total { + archives + artists + characters + circles + comics + namespaces + scrapers + tags + worlds + images + pages + comic { + artists + characters + circles + tags + worlds + } + } + } +""" + + +@pytest.fixture +def query_statistics(execute): + query = """ + query statistics { + statistics { + __typename + ... Totals + } + } + """ + + return execute(totals_fragment + query) + + +@pytest.mark.anyio +async def test_statistics_returns_totals( + gen_comic, gen_image, query_statistics, empty_plugins +): + comic = next(gen_comic) + await DB.add(comic) + await DB.add(Artist(name="foo")) + await DB.add(Character(name="foo")) + await DB.add(Circle(name="foo")) + await DB.add(World(name="foo")) + await DB.add(Tag(name="foo")) + + image = next(gen_image) + await DB.add(image) + await DB.add( + Page(id=100, index=100, path="100.png", image=image, archive=comic.archive) + ) + await DB.add( + Page(id=101, index=101, path="101.png", image=image, archive=comic.archive) + ) + + namespaces = set() + for tag in comic.tags: + namespaces.add(tag.namespace.id) + + class MockScraper(Scraper): + name = "Scraper" + + def scrape(self): + yield None + + hircine.plugins.register_scraper("mock", MockScraper) + + response = Response(await query_statistics()) + + response.assert_is("Statistics") + assert response.total["comics"] == 1 + assert response.total["archives"] == 1 + assert response.total["artists"] == len(comic.artists) + 1 + assert response.total["characters"] == len(comic.characters) + 1 + assert response.total["circles"] == len(comic.circles) + 1 + assert response.total["worlds"] == len(comic.worlds) + 1 + assert response.total["tags"] == len(comic.tags) + 1 + assert response.total["namespaces"] == len(namespaces) + assert response.total["images"] == len(comic.pages) + 1 + assert response.total["pages"] == len(comic.pages) + 2 + assert response.total["scrapers"] == 1 + assert response.total["comic"]["artists"] == len(comic.artists) + assert response.total["comic"]["characters"] == len(comic.characters) + assert response.total["comic"]["circles"] == len(comic.circles) + assert response.total["comic"]["tags"] == len(comic.tags) + assert response.total["comic"]["worlds"] == len(comic.worlds) diff --git a/tests/api/test_tag.py b/tests/api/test_tag.py index c863a00..7970f3d 100644 --- a/tests/api/test_tag.py +++ b/tests/api/test_tag.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import Namespace, Tag diff --git a/tests/api/test_world.py b/tests/api/test_world.py index a3926d1..3546c2c 100644 --- a/tests/api/test_world.py +++ b/tests/api/test_world.py @@ -3,6 +3,7 @@ from datetime import timezone import pytest from conftest import DB, Response + from hircine.db.models import World diff --git a/tests/conftest.py b/tests/conftest.py index a36be2d..d831827 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,14 +4,15 @@ from datetime import date, timedelta from datetime import datetime as dt from datetime import timezone as tz +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + import hircine import hircine.db as database import hircine.db.models as models import hircine.plugins -import pytest from hircine.app import schema from hircine.enums import Category, Censorship, Direction, Language, Layout, Rating -from sqlalchemy.ext.asyncio import AsyncSession @pytest.fixture(scope="session") diff --git a/tests/plugins/scrapers/test_anchira.py b/tests/plugins/scrapers/test_anchira.py new file mode 100644 index 0000000..0554957 --- /dev/null +++ b/tests/plugins/scrapers/test_anchira.py @@ -0,0 +1,107 @@ +import os +from datetime import date +from zipfile import ZipFile + +import pytest + +import hircine.enums as enums +from hircine.plugins.scrapers.anchira import AnchiraYamlScraper +from hircine.scraper.types import ( + URL, + Artist, + Censorship, + Circle, + Date, + Direction, + Language, + Rating, + Tag, + Title, + World, +) + + +@pytest.fixture +def archive_file(tmpdir): + file = os.path.join(tmpdir, "archive.zip") + + data = """ +--- +Source: https://anchira.to/g/1/1 +URL: https://example.com +Title: Example Title +Artist: +- Example +Circle: +- Example +Parody: +- Original Work +- Example +Magazine: [] +Tags: +- Unlimited +- Book +Released: 1574394240 +Pages: 102 +... + """ + + with ZipFile(file, "x") as ziph: + ziph.writestr("info.yaml", data) + + yield file + + +def test_does_scrape(monkeypatch, archive_file, gen_comic): + comic = next(gen_comic) + comic.archive.path = archive_file + + scraper = AnchiraYamlScraper(comic) + + assert scraper.is_available + assert scraper.source == AnchiraYamlScraper.source + assert scraper.name == "anchira.to info.yaml" + + assert set(scraper.collect()) == set( + [ + Artist(name="Example"), + Circle(name="Example"), + Date(value=date(2019, 11, 22)), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + Language(value=enums.Language.EN), + Tag(namespace="none", tag="Book"), + Title(value="Example Title"), + URL(value="https://example.com"), + World(name="Example"), + ] + ) + + +def test_does_not_scrape_on_error(tmpdir, gen_comic): + comic = next(gen_comic) + comic.archive.path = os.path.join(tmpdir, "nonexistent.zip") + + scraper = AnchiraYamlScraper(comic) + + assert scraper.data == {} + assert not scraper.is_available + + +@pytest.mark.parametrize( + "tag, parsed", + [ + ("Hentai", Rating(value=enums.Rating.EXPLICIT)), + ("Non-H", Rating(value=enums.Rating.QUESTIONABLE)), + ("Ecchi", Rating(value=enums.Rating.QUESTIONABLE)), + ("Uncensored", Censorship(value=enums.Censorship.NONE)), + ], + ids=[ + "hentai", + "non-h", + "ecchi", + "uncensored", + ], +) +def test_parses_tags(tag, parsed): + scraper = AnchiraYamlScraper(None) + assert scraper.parse_tag(tag) == parsed diff --git a/tests/plugins/scrapers/test_ehentai_api.py b/tests/plugins/scrapers/test_ehentai_api.py new file mode 100644 index 0000000..c746440 --- /dev/null +++ b/tests/plugins/scrapers/test_ehentai_api.py @@ -0,0 +1,156 @@ +from datetime import date + +import pytest + +import hircine.enums as enums +import hircine.plugins.scrapers.ehentai_api as ehentai_api +from hircine.scraper import ScrapeError +from hircine.scraper.types import ( + URL, + Artist, + Censorship, + Circle, + Date, + Language, + OriginalTitle, + Rating, + Tag, + Title, + World, +) + + +def test_does_scrape(requests_mock, gen_comic): + comic = next(gen_comic) + comic.url = "https://exhentai.org/g/1025913/fdaabef1a2" + + scraper = ehentai_api.EHentaiAPIScraper(comic) + + requests_mock.post( + ehentai_api.API_URL, + text=""" + { + "gmetadata": [ + { + "gid": 1025913, + "token": "fdaabef1a2", + "title": "(C91) [Animachine (Shimahara)] Iya na Kao Sarenagara Opantsu Misete Moraitai Manga | A manga about girl showing you her panties while making a disgusted face [English] [葛の寺]", + "title_jpn": "(C91) [アニマルマシーン (40原)] 嫌な顔されながらおパンツ見せてもらいたい漫画 [英訳]", + "category": "Non-H", + "thumb": "https://ehgt.org/51/17/5117cde63cc14436c5ad7f2dd06abb52c86aff65-23642001-2866-4047-png_250.jpg", + "uploader": "葛の寺", + "posted": "1486182349", + "filecount": "23", + "filesize": 528093263, + "expunged": false, + "rating": "4.72", + "torrentcount": "1", + "torrents": [ + { + "hash": "30c7124efca83bf0db1b9fd5ab4511da5f28a60b", + "added": "1486121301", + "name": "(C91) [Animachine (Shimahara)] Iya na Kao Sarenagara Opantsu Misete Moraitai Manga A manga about girl showing you her panties while making a disgusted face [English] [葛の寺].zip", + "tsize": "20800", + "fsize": "528093242" + } + ], + "tags": [ + "language:english", + "language:translated", + "parody:iya na kao sare nagara opantsu misete moraitai", + "group:animachine", + "artist:shimahara", + "female:femdom", + "female:schoolgirl uniform", + "other:full color" + ], + "parent_gid": "1025875", + "parent_key": "cfe6adccb8", + "first_gid": "1025646", + "first_key": "098b4a982a" + } + ] + } + """, # noqa: E501 + ) + + assert scraper.is_available + assert scraper.id == 1025913 + assert scraper.token == "fdaabef1a2" + + assert set(scraper.collect()) == set( + [ + Artist(name="shimahara"), + Censorship(value=enums.Censorship.NONE), + Circle(name="animachine"), + Date(value=date(2017, 2, 4)), + Language(value=enums.Language.EN), + OriginalTitle(value="嫌な顔されながらおパンツ見せてもらいたい漫画"), + Rating(value=enums.Rating.QUESTIONABLE), + Tag(namespace="female", tag="femdom"), + Tag(namespace="female", tag="schoolgirl uniform"), + Tag(namespace="other", tag="full color"), + Title( + value="A manga about girl showing you her panties while making a disgusted face" # noqa: E501 + ), + URL("https://exhentai.org/g/1025913/fdaabef1a2"), + World(name="iya na kao sare nagara opantsu misete moraitai"), + ] + ) + + +def test_is_not_available_with_wrong_url(gen_comic): + comic = next(gen_comic) + comic.url = "https://example.com" + + scraper = ehentai_api.EHentaiAPIScraper(comic) + + assert not scraper.is_available + + +def test_raises_scrape_error_with_invalid_json(requests_mock, gen_comic): + comic = next(gen_comic) + comic.url = "https://exhentai.org/g/1025913/fdaabef1a2" + + scraper = ehentai_api.EHentaiAPIScraper(comic) + + requests_mock.post(ehentai_api.API_URL, text="{") + + assert scraper.is_available + assert scraper.id == 1025913 + assert scraper.token == "fdaabef1a2" + + with pytest.raises(ScrapeError, match="Could not parse JSON response"): + assert set(scraper.collect()) == set() + + +def test_raises_scrape_error_with_missing_field(requests_mock, gen_comic): + comic = next(gen_comic) + comic.url = "https://exhentai.org/g/1025913/fdaabef1a2" + + scraper = ehentai_api.EHentaiAPIScraper(comic) + + requests_mock.post(ehentai_api.API_URL, text="{}") + + assert scraper.is_available + assert scraper.id == 1025913 + assert scraper.token == "fdaabef1a2" + + with pytest.raises(ScrapeError, match="Response is missing 'gmetadata' field"): + assert set(scraper.collect()) == set() + + +def test_raises_scrape_error_with_error_code(requests_mock, gen_comic): + comic = next(gen_comic) + comic.url = "https://exhentai.org/g/1025913/fdaabef1a2" + + scraper = ehentai_api.EHentaiAPIScraper(comic) + + requests_mock.post(ehentai_api.API_URL, status_code=500) + + assert scraper.is_available + assert scraper.id == 1025913 + assert scraper.token == "fdaabef1a2" + + with pytest.raises(ScrapeError, match="Request failed with status code 500"): + assert set(scraper.collect()) == set() diff --git a/tests/plugins/scrapers/test_gallery_dl.py b/tests/plugins/scrapers/test_gallery_dl.py new file mode 100644 index 0000000..f5bdb88 --- /dev/null +++ b/tests/plugins/scrapers/test_gallery_dl.py @@ -0,0 +1,52 @@ +import json +import os +from zipfile import ZipFile + +import pytest + +import hircine.plugins.scrapers.gallery_dl +from hircine.plugins.scrapers.gallery_dl import GalleryDLScraper +from hircine.scraper.types import Title + + +class MockHandler: + source = "mock" + + def scrape(self, data): + yield Title(data["title"]) + + +@pytest.fixture +def archive_file(tmpdir): + file = os.path.join(tmpdir, "archive.zip") + + with ZipFile(file, "x") as ziph: + ziph.writestr("info.json", json.dumps({"category": "mock", "title": "test"})) + + yield file + + +def test_does_scrape(monkeypatch, archive_file, gen_comic): + comic = next(gen_comic) + comic.archive.path = archive_file + + monkeypatch.setattr( + hircine.plugins.scrapers.gallery_dl, "HANDLERS", {"mock": MockHandler} + ) + + scraper = GalleryDLScraper(comic) + + assert scraper.is_available + assert scraper.source == MockHandler.source + assert scraper.name == f"gallery-dl info.json ({MockHandler.source})" + assert set(scraper.collect()) == set([Title(value="test")]) + + +def test_does_not_scrape_on_error(tmpdir, gen_comic): + comic = next(gen_comic) + comic.archive.path = os.path.join(tmpdir, "nonexistent.zip") + + scraper = GalleryDLScraper(comic) + + assert scraper.data == {} + assert not scraper.is_available diff --git a/tests/plugins/scrapers/test_handlers.py b/tests/plugins/scrapers/test_handlers.py new file mode 100644 index 0000000..4e71041 --- /dev/null +++ b/tests/plugins/scrapers/test_handlers.py @@ -0,0 +1,597 @@ +import json +from datetime import date + +import pytest + +import hircine.enums as enums +from hircine.plugins.scrapers.handlers.dynastyscans import DynastyScansHandler +from hircine.plugins.scrapers.handlers.e621 import E621Handler +from hircine.plugins.scrapers.handlers.exhentai import ( + ExHentaiHandler, +) +from hircine.plugins.scrapers.handlers.exhentai import ( + sanitize as exhentai_sanitize, +) +from hircine.plugins.scrapers.handlers.mangadex import MangadexHandler +from hircine.scraper import Scraper +from hircine.scraper.types import ( + URL, + Artist, + Category, + Censorship, + Character, + Circle, + Date, + Direction, + Language, + OriginalTitle, + Rating, + Tag, + Title, + World, +) + + +class Scraper(Scraper): + def __init__(self, handler, json): + self.handler = handler + self.json = json + super().__init__(None) + + def scrape(self): + yield from self.handler.scrape(json.loads(self.json)) + + +def test_dynastyscans(): + scraper = Scraper( + DynastyScansHandler(), + """ + { + "manga": "Hoshiiro GirlDrop Comic Anthology", + "chapter": 1, + "chapter_minor": "", + "title": "Hop, Step, Drop!", + "author": "Fujisawa Kamiya", + "group": "Cyan Steam (Stan Miller)", + "date": "2018-02-05 00:00:00", + "lang": "en", + "language": "English", + "count": 15, + "category": "dynastyscans", + "subcategory": "manga" + } + """, + ) + + assert set(scraper.collect()) == set( + [ + Artist(name="Fujisawa Kamiya"), + Circle(name="Cyan Steam (Stan Miller)"), + Date(value=date(2018, 2, 5)), + Language(value=enums.Language.EN), + Title(value="Hoshiiro GirlDrop Comic Anthology Ch. 1: Hop, Step, Drop!"), + ] + ) + + +def test_mangadex(): + scraper = Scraper( + MangadexHandler(), + """ + { + "manga": "Shimeji Simulation", + "manga_id": "28b5d037-175d-4119-96f8-e860e408ebe9", + "title": "Danchi", + "volume": 1, + "chapter": 8, + "chapter_minor": "", + "chapter_id": "2a115ccb-de52-4b84-9166-cebd152d9396", + "date": "2019-09-22 04:19:15", + "lang": "en", + "language": "English", + "count": 12, + "artist": [ + "Tsukumizu" + ], + "author": [ + "Tsukumizu" + ], + "group": [ + "Orchesc/a/ns" + ], + "status": "completed", + "tags": [ + "Sci-Fi", + "Comedy", + "Girls' Love", + "4-Koma", + "Philosophical", + "School Life", + "Slice of Life" + ], + "category": "mangadex", + "subcategory": "chapter" + } + """, + ) + + assert set(scraper.collect()) == set( + [ + Artist(name="Tsukumizu"), + Circle(name="Orchesc/a/ns"), + Date(value=date(2019, 9, 22)), + Language(value=enums.Language.EN), + Tag(namespace="none", tag="4-Koma"), + Tag(namespace="none", tag="Comedy"), + Tag(namespace="none", tag="Girls' Love"), + Tag(namespace="none", tag="Philosophical"), + Tag(namespace="none", tag="School Life"), + Tag(namespace="none", tag="Sci-Fi"), + Tag(namespace="none", tag="Slice of Life"), + Title(value="Shimeji Simulation Vol. 1, Ch. 8: Danchi"), + URL("https://mangadex.org/chapter/2a115ccb-de52-4b84-9166-cebd152d9396"), + ] + ) + + +@pytest.mark.parametrize( + "data, title", + [ + ({"volume": 1, "chapter": 8}, "Manga Vol. 1, Ch. 8: Title"), + ({"volume": 0, "chapter": 1}, "Manga Ch. 1: Title"), + ({"volume": 0, "chapter": 0}, "Manga: Title"), + ], + ids=[ + "volume and chapter", + "chapter only", + "none", + ], +) +def test_mangadex_handles_volume_and_chapter(data, title): + common = {"manga": "Manga", "title": "Title"} + scraper = Scraper(MangadexHandler(), json.dumps(common | data)) + + assert list(scraper.collect()) == [Title(value=title)] + + +def test_e621_pool(): + scraper = Scraper( + E621Handler(), + """ + { + "id": 2968472, + "created_at": "2021-10-10T04:13:53.286-04:00", + "updated_at": "2024-11-02T08:58:06.724-04:00", + "file": { + "width": 800, + "height": 800, + "ext": "jpg", + "size": 530984, + "md5": "1ec7e397bb22c1454ab1986fd3f3edc5", + "url": "https://static1.e621.net/data/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg" + }, + "preview": { + "width": 150, + "height": 150, + "url": "https://static1.e621.net/data/preview/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg" + }, + "sample": { + "has": false, + "height": 800, + "width": 800, + "url": "https://static1.e621.net/data/1e/c7/1ec7e397bb22c1454ab1986fd3f3edc5.jpg", + "alternates": {} + }, + "score": { + "up": 202, + "down": -1, + "total": 201 + }, + "tags": { + "general": [ + "beak" + ], + "artist": [ + "falseknees" + ], + "copyright": [], + "character": [], + "species": [ + "bird" + ], + "invalid": [], + "meta": [ + "comic", + "english_text" + ], + "lore": [ + "parent_(lore)", + "parent_and_child_(lore)" + ] + }, + "locked_tags": [], + "change_seq": 60808337, + "flags": { + "pending": false, + "flagged": false, + "note_locked": false, + "status_locked": false, + "rating_locked": false, + "deleted": false + }, + "rating": "s", + "fav_count": 194, + "sources": [ + "https://twitter.com/FalseKnees/status/1324869853627478022" + ], + "pools": [ + 25779 + ], + "relationships": { + "parent_id": null, + "has_children": false, + "has_active_children": false, + "children": [] + }, + "approver_id": 171673, + "uploader_id": 178921, + "description": "", + "comment_count": 1, + "is_favorited": false, + "has_notes": false, + "duration": null, + "num": 1, + "filename": "1ec7e397bb22c1454ab1986fd3f3edc5", + "extension": "jpg", + "date": "2021-10-10 08:13:53", + "pool": { + "id": 25779, + "name": "Kids say the darnedest shit - falseknees", + "created_at": "2021-10-10T04:17:07.006-04:00", + "updated_at": "2021-10-10T04:17:07.006-04:00", + "creator_id": 178921, + "description": "The terror of every parent.", + "is_active": true, + "category": "series", + "creator_name": "OneMoreAnonymous", + "post_count": 4 + }, + "category": "e621", + "subcategory": "pool" + } + """, + ) + + assert set(scraper.collect()) == set( + [ + Artist(name="falseknees"), + Category(value=enums.Category.COMIC), + Censorship(value=enums.Censorship.NONE), + Date(value=date(2021, 10, 10)), + Language(value=enums.Language.EN), + Rating(value=enums.Rating.SAFE), + Tag(namespace="none", tag="beak"), + Tag(namespace="none", tag="bird"), + Title(value="Kids say the darnedest shit - falseknees"), + URL("https://e621.net/pools/25779"), + ] + ) + + +@pytest.mark.parametrize( + "data, censorship", + [ + ({"tags": {"meta": ["censor_bar"]}}, enums.Censorship.BAR), + ({"tags": {"meta": ["mosaic_censorship"]}}, enums.Censorship.MOSAIC), + ({"tags": {"meta": ["uncensored"]}}, enums.Censorship.NONE), + ({"tags": {"meta": []}}, enums.Censorship.NONE), + ], + ids=[ + "bars", + "mosaic", + "uncensored", + "uncensored (implied)", + ], +) +def test_e621_handles_censorship(data, censorship): + common = {"subcategory": "pool"} + scraper = Scraper(E621Handler(), json.dumps(common | data)) + + assert set(scraper.collect()) == set([Censorship(value=censorship)]) + + +def test_exhentai_explicit(): + scraper = Scraper( + ExHentaiHandler(), + """ + { + "gid": 2771624, + "token": "43108ee23b", + "thumb": "https://s.exhentai.org/t/12/80/1280a064a2ab3d70b9feb56bd0c55dbfc3ab6a39-309830-950-1351-jpg_250.jpg", + "title": "[NAGABE] Smell ch.01 - ch.06", + "title_jpn": "SMELL", + "eh_category": "Doujinshi", + "uploader": "randaldog", + "date": "2023-12-19 23:50:00", + "parent": "https://exhentai.org/g/2736803/b191bfed72/", + "expunged": false, + "language": "English", + "filesize": 74469868, + "filecount": "170", + "favorites": "751", + "rating": "4.83", + "torrentcount": "0", + "lang": "en", + "tags": [ + "language:english", + "language:translated", + "parody:original", + "artist:nagabe", + "male:dog boy", + "male:furry", + "male:males only", + "male:smell", + "male:yaoi", + "other:story arc" + ], + "category": "exhentai", + "subcategory": "gallery" + } + """, + ) + + assert set(scraper.collect()) == set( + [ + Artist(name="nagabe"), + Category(value=enums.Category.DOUJINSHI), + Censorship(value=enums.Censorship.BAR), + Date(value=date(2023, 12, 19)), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + Language(value=enums.Language.EN), + OriginalTitle(value="SMELL"), + Rating(value=enums.Rating.EXPLICIT), + Tag(namespace="male", tag="dog boy"), + Tag(namespace="male", tag="furry"), + Tag(namespace="male", tag="males only"), + Tag(namespace="male", tag="smell"), + Tag(namespace="male", tag="yaoi"), + Tag(namespace="other", tag="story arc"), + Title(value="Smell ch.01 - ch.06"), + URL("https://exhentai.org/g/2771624/43108ee23b"), + World(name="original"), + ] + ) + + +def test_exhentai_non_h(): + scraper = Scraper( + ExHentaiHandler(), + """ + { + "gid": 1025913, + "token": "fdaabef1a2", + "thumb": "https://s.exhentai.org/t/51/17/5117cde63cc14436c5ad7f2dd06abb52c86aff65-23642001-2866-4047-png_250.jpg", + "title": "(C91) [Animachine (Shimahara)] Iya na Kao Sarenagara Opantsu Misete Moraitai Manga | A manga about girl showing you her panties while making a disgusted face [English] [葛の寺]", + "title_jpn": "(C91) [アニマルマシーン (40原)] 嫌な顔されながらおパンツ見せてもらいたい漫画 [英訳]", + "eh_category": "Non-H", + "uploader": "葛の寺", + "date": "2017-02-04 04:25:00", + "parent": "https://exhentai.org/g/1025875/cfe6adccb8/", + "expunged": false, + "language": "English", + "filesize": 0, + "filecount": "23", + "favorites": "1088", + "rating": "4.74", + "torrentcount": "1", + "lang": "en", + "tags": [ + "language:english", + "language:translated", + "parody:iya na kao sare nagara opantsu misete moraitai", + "group:animachine", + "artist:shimahara", + "female:femdom", + "female:schoolgirl uniform", + "other:full color" + ], + "category": "exhentai", + "subcategory": "gallery" + } + """, # noqa: E501 + ) + + assert set(scraper.collect()) == set( + [ + Artist(name="shimahara"), + Censorship(value=enums.Censorship.NONE), + Circle(name="animachine"), + Date(value=date(2017, 2, 4)), + Language(value=enums.Language.EN), + OriginalTitle(value="嫌な顔されながらおパンツ見せてもらいたい漫画"), + Rating(value=enums.Rating.QUESTIONABLE), + Tag(namespace="female", tag="femdom"), + Tag(namespace="female", tag="schoolgirl uniform"), + Tag(namespace="other", tag="full color"), + Title( + value="A manga about girl showing you her panties while making a disgusted face" # noqa: E501 + ), + URL("https://exhentai.org/g/1025913/fdaabef1a2"), + World(name="iya na kao sare nagara opantsu misete moraitai"), + ] + ) + + +@pytest.mark.parametrize( + "text, sanitized", + [ + ("(foo) Title", "Title"), + ("[foo] {bar} =baz= Title", "Title"), + ("Foreign Title | Localized Title", "Localized Title"), + ], + ids=[ + "parens at beginning", + "bracket-likes", + "split titles", + ], +) +def test_exhentai_sanitizes(text, sanitized): + assert exhentai_sanitize(text, split=True) == sanitized + + +@pytest.mark.parametrize( + "data, expect", + [ + ( + {"category": "doujinshi"}, + set( + [ + Category(value=enums.Category.DOUJINSHI), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + ] + ), + ), + ( + {"eh_category": "doujinshi"}, + set( + [ + Category(value=enums.Category.DOUJINSHI), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + ] + ), + ), + ( + {"category": "manga"}, + set( + [ + Category(value=enums.Category.MANGA), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + ] + ), + ), + ( + {"category": "western"}, + set( + [ + Censorship(value=enums.Censorship.NONE), + Rating(value=enums.Rating.EXPLICIT), + ] + ), + ), + ( + {"category": "artist cg"}, + set( + [ + Category(value=enums.Category.COMIC), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + ] + ), + ), + ( + {"category": "game cg"}, + set( + [ + Category(value=enums.Category.GAME_CG), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + ] + ), + ), + ( + {"category": "image set"}, + set( + [ + Category(value=enums.Category.IMAGE_SET), + Censorship(value=enums.Censorship.BAR), + Rating(value=enums.Rating.EXPLICIT), + ] + ), + ), + ( + {"category": "non-h"}, + set( + [ + Censorship(value=enums.Censorship.NONE), + Rating(value=enums.Rating.QUESTIONABLE), + ] + ), + ), + ( + {"category": "western", "tags": ["other:western non-h"]}, + set( + [ + Censorship(value=enums.Censorship.NONE), + Rating(value=enums.Rating.QUESTIONABLE), + ] + ), + ), + ], + ids=[ + "category from category field", + "category from eh_category field", + "manga category", + "western category", + "artist cg category", + "game cg category", + "image set category", + "non-h category", + "western non-h tag", + ], +) +def test_exhentai_parses(data, expect): + scraper = Scraper(ExHentaiHandler(), json.dumps(data | {"gid": 1, "token": 1})) + + expect.add(URL(value="https://exhentai.org/g/1/1")) + + assert set(scraper.collect()) == expect + + +@pytest.mark.parametrize( + "tag, parsed", + [ + ("parody:foo", World(name="foo")), + ("artist:foo", Artist(name="foo")), + ("character:foo", Character(name="foo")), + ("group:foo", Circle(name="foo")), + ("other:artbook", Category(value=enums.Category.ARTBOOK)), + ("other:non-h imageset", Category(value=enums.Category.IMAGE_SET)), + ("other:western imageset", Category(value=enums.Category.IMAGE_SET)), + ("other:comic", Category(value=enums.Category.COMIC)), + ("other:variant set", Category(value=enums.Category.VARIANT_SET)), + ("other:webtoon", Category(value=enums.Category.WEBTOON)), + ("other:full censorship", Censorship(value=enums.Censorship.FULL)), + ("other:mosaic censorship", Censorship(value=enums.Censorship.MOSAIC)), + ("other:uncensored", Censorship(value=enums.Censorship.NONE)), + ("generic", Tag(namespace=None, tag="generic")), + ], + ids=[ + "parody", + "group", + "artist", + "character", + "other:artbook", + "other:image set", + "other:western image set", + "other:comic", + "other:variant set", + "other:webtoon", + "other:full censorship", + "other:mosaic censorship", + "other:uncensored", + "generic", + ], +) +def test_exhentai_parses_tags(tag, parsed): + scraper = Scraper( + ExHentaiHandler(), json.dumps({"tags": [tag], "gid": 1, "token": 1}) + ) + expect = set([URL(value="https://exhentai.org/g/1/1"), parsed]) + + assert set(scraper.collect()) > expect diff --git a/tests/plugins/scrapers/test_schale_network.py b/tests/plugins/scrapers/test_schale_network.py new file mode 100644 index 0000000..236520b --- /dev/null +++ b/tests/plugins/scrapers/test_schale_network.py @@ -0,0 +1,88 @@ +import os +from zipfile import ZipFile + +import pytest + +import hircine.enums as enums +from hircine.plugins.scrapers.schale_network import SchaleNetworkScraper +from hircine.scraper.types import ( + Artist, + Censorship, + Circle, + Direction, + Language, + Tag, + Title, +) + + +@pytest.fixture +def archive_file(tmpdir): + file = os.path.join(tmpdir, "archive.zip") + + data = """ +source: SchaleNetwork:/g/1/1 +title: 'Example Title' +general: + - example +artist: + - example +circle: + - example +magazine: + - example +male: + - example +female: + - example +mixed: + - example +language: + - english + - translated +other: + - uncensored + - vanilla +""" + + with ZipFile(file, "x") as ziph: + ziph.writestr("info.yaml", data) + + yield file + + +def test_does_scrape(monkeypatch, archive_file, gen_comic): + comic = next(gen_comic) + comic.archive.path = archive_file + + scraper = SchaleNetworkScraper(comic) + + assert scraper.is_available + assert scraper.source == SchaleNetworkScraper.source + assert scraper.name == "schale.network info.yaml" + + assert set(scraper.collect()) == set( + [ + Artist(name="example"), + Circle(name="example"), + Direction(value=enums.Direction.RIGHT_TO_LEFT), + Censorship(value=enums.Censorship.NONE), + Language(value=enums.Language.EN), + Tag(namespace="none", tag="example"), + Tag(namespace="none", tag="vanilla"), + Tag(namespace="male", tag="example"), + Tag(namespace="female", tag="example"), + Tag(namespace="mixed", tag="example"), + Title(value="Example Title"), + ] + ) + + +def test_does_not_scrape_on_error(tmpdir, gen_comic): + comic = next(gen_comic) + comic.archive.path = os.path.join(tmpdir, "nonexistent.zip") + + scraper = SchaleNetworkScraper(comic) + + assert scraper.data == {} + assert not scraper.is_available diff --git a/tests/scanner/data/bad/bad_compression.zip b/tests/scanner/data/bad/bad_compression.zip Binary files differnew file mode 100644 index 0000000..4dbbc1f --- /dev/null +++ b/tests/scanner/data/bad/bad_compression.zip diff --git a/tests/scanner/data/bad/bad_entry.zip b/tests/scanner/data/bad/bad_entry.zip Binary files differnew file mode 100644 index 0000000..0bf6e13 --- /dev/null +++ b/tests/scanner/data/bad/bad_entry.zip diff --git a/tests/scanner/test_scanner.py b/tests/scanner/test_scanner.py index 45a966f..141698c 100644 --- a/tests/scanner/test_scanner.py +++ b/tests/scanner/test_scanner.py @@ -3,11 +3,12 @@ import os import shutil from datetime import datetime, timezone from pathlib import Path -from zipfile import ZipFile +from zipfile import BadZipFile, ZipFile -import hircine.thumbnailer import pytest from conftest import DB + +import hircine.thumbnailer from hircine.config import DirectoryStructure from hircine.db.models import Archive, Image, Page from hircine.scanner import Scanner, Status @@ -103,18 +104,17 @@ async def test_scanner_dedups_archive_contents(archive, scanner, capsys): archive = await DB.add(archive) dedup_path = archive.path + ".dedup" - with ZipFile(archive.path, "r") as zin: - with ZipFile(dedup_path, "w") as zout: - for info in zin.infolist(): - base, ext = os.path.splitext(info.filename) + with ZipFile(archive.path, "r") as zin, ZipFile(dedup_path, "w") as zout: + for info in zin.infolist(): + base, ext = os.path.splitext(info.filename) - if base == "03": - continue + if base == "03": + continue - if ext == ".png": - zout.writestr(f"0{base}.png", zin.read(info)) - else: - zout.writestr(info.filename, zin.read(info)) + if ext == ".png": + zout.writestr(f"0{base}.png", zin.read(info)) + else: + zout.writestr(info.filename, zin.read(info)) await scanner.scan() added_archive = await DB.get(Archive, 2, full=True) @@ -309,3 +309,19 @@ async def test_scanner_reprocess(archive, data, scanner, capsys): captured = capsys.readouterr() assert captured.out == "[~] archive.zip\n" + + +@pytest.mark.anyio +async def test_scanner_handles_bad_zip_entry(data, scanner): + Path(data("bad/bad_entry.zip")).rename(data("contents/bad_entry.zip")) + + with pytest.raises(BadZipFile): + await scanner.scan() + + +@pytest.mark.anyio +async def test_scanner_handles_bad_zip_compression(data, scanner): + Path(data("bad/bad_compression.zip")).rename(data("contents/bad_compression.zip")) + + with pytest.raises(BadZipFile): + await scanner.scan() diff --git a/tests/scrapers/test_scraper.py b/tests/scrapers/test_scraper.py index 6f6f29d..d0cef7b 100644 --- a/tests/scrapers/test_scraper.py +++ b/tests/scrapers/test_scraper.py @@ -9,10 +9,18 @@ class MockScraper(Scraper): yield "bar" +class NoneScraper(Scraper): + is_available = True + + def scrape(self): + yield lambda: "foo" + yield None + + class WarningScraper(Scraper): is_available = True - def warn(self, str): + def warn(self, msg): raise ScrapeWarning("Invalid input") def scrape(self): @@ -53,3 +61,10 @@ def test_scraper_collects_literal(): generator = scraper.collect() assert set(generator) == set(["literal"]) + + +def test_scraper_collect_ignores_none(): + scraper = NoneScraper(None) + generator = scraper.collect() + + assert set(generator) == set(["foo"]) diff --git a/tests/scrapers/test_scraper_utils.py b/tests/scrapers/test_scraper_utils.py index 193cf2a..30b9796 100644 --- a/tests/scrapers/test_scraper_utils.py +++ b/tests/scrapers/test_scraper_utils.py @@ -1,24 +1,30 @@ -from hircine.scraper.utils import parse_dict +import json +import os +from zipfile import ZipFile + +import pytest + +from hircine.scraper.utils import open_archive_file, parse_dict def test_parse_dict(): - dict = { + data = { "scalar": "foo", "list": ["bar", "baz"], "dict": {"nested_scalar": "qux", "nested_list": ["plugh", "xyzzy"]}, } - def id(type): - return lambda item: f"{type}_{item}" + def annotate(tag): + return lambda item: f"{tag}_{item}" parsers = { - "scalar": id("scalar"), - "list": id("list"), - "dict": {"nested_scalar": id("scalar"), "nested_list": id("list")}, - "missing": id("missing"), + "scalar": annotate("scalar"), + "list": annotate("list"), + "dict": {"nested_scalar": annotate("scalar"), "nested_list": annotate("list")}, + "missing": annotate("missing"), } - assert [f() for f in parse_dict(parsers, dict)] == [ + assert [f() for f in parse_dict(parsers, data)] == [ "scalar_foo", "list_bar", "list_baz", @@ -26,3 +32,36 @@ def test_parse_dict(): "list_plugh", "list_xyzzy", ] + + +@pytest.mark.parametrize( + "check_sidecar", + [ + (False), + (True), + ], + ids=[ + "zip", + "sidecar", + ], +) +def test_open_archive_file(gen_archive, tmpdir, check_sidecar): + archive = next(gen_archive) + archive.path = os.path.join(tmpdir, "archive.zip") + + zip_data = {"zip": "data"} + sidecar_data = {"sidecar": "data"} + + with open(f"{archive.path}.info.json", "x") as handle: + json.dump(sidecar_data, handle) + + with ZipFile(archive.path, "x") as ziph: + ziph.writestr("info.json", json.dumps(zip_data)) + + with open_archive_file(archive, "info.json", check_sidecar=check_sidecar) as file: + data = json.load(file) + + if check_sidecar: + assert data == sidecar_data + else: + assert data == zip_data diff --git a/tests/scrapers/test_types.py b/tests/scrapers/test_types.py index ed937e7..33f9f89 100644 --- a/tests/scrapers/test_types.py +++ b/tests/scrapers/test_types.py @@ -1,6 +1,8 @@ from datetime import date import pytest + +import hircine.enums as enums from hircine.api.types import ScrapedComic from hircine.scraper import ScrapeWarning from hircine.scraper.types import ( @@ -129,3 +131,35 @@ def test_scraped_comic_silently_ignores_empty(item, attr, empty): comic = ScrapedComic.from_generator(gen()) assert getattr(comic, attr) == empty + + +@pytest.mark.parametrize( + "input,want", + [ + ("EN", Language(value=enums.Language.EN)), + ("de", Language(value=enums.Language.DE)), + ], +) +def test_language_from_iso_639_3(input, want): + assert Language.from_iso_639_3(input) == want + + +def test_language_from_iso_639_3_fails(): + with pytest.raises(ScrapeWarning, match="Could not parse language code:"): + Language.from_iso_639_3("ENG") + + +@pytest.mark.parametrize( + "input,want", + [ + ("English", Language(value=enums.Language.EN)), + ("german", Language(value=enums.Language.DE)), + ], +) +def test_language_from_name(input, want): + assert Language.from_name(input) == want + + +def test_language_from_name_fails(): + with pytest.raises(ScrapeWarning, match="Could not parse language name:"): + Language.from_name("nonexistent") diff --git a/tests/thumbnailer/test_thumbnailer.py b/tests/thumbnailer/test_thumbnailer.py index 62bf127..3a3405a 100644 --- a/tests/thumbnailer/test_thumbnailer.py +++ b/tests/thumbnailer/test_thumbnailer.py @@ -2,9 +2,10 @@ import os from pathlib import Path import pytest -from hircine.thumbnailer import Thumbnailer, ThumbnailParameters from PIL import Image +from hircine.thumbnailer import Thumbnailer, ThumbnailParameters + mock_params = ThumbnailParameters(bounds=(1440, 2880), options={}) |