summaryrefslogtreecommitdiffstatshomepage
path: root/frontend/src/lib/scraper
diff options
context:
space:
mode:
Diffstat (limited to 'frontend/src/lib/scraper')
-rw-r--r--frontend/src/lib/scraper/ComicScrapeForm.svelte125
-rw-r--r--frontend/src/lib/scraper/Scraper.svelte.ts164
-rw-r--r--frontend/src/lib/scraper/components/SelectorButton.svelte10
-rw-r--r--frontend/src/lib/scraper/components/SelectorGroup.svelte13
-rw-r--r--frontend/src/lib/scraper/components/SelectorItem.svelte5
5 files changed, 247 insertions, 70 deletions
diff --git a/frontend/src/lib/scraper/ComicScrapeForm.svelte b/frontend/src/lib/scraper/ComicScrapeForm.svelte
index 30ad89b..6cc3451 100644
--- a/frontend/src/lib/scraper/ComicScrapeForm.svelte
+++ b/frontend/src/lib/scraper/ComicScrapeForm.svelte
@@ -2,60 +2,69 @@
import { upsertComics } from '$gql/Mutations';
import { comicScrapersQuery, scrapeComic } from '$gql/Queries';
import { isError } from '$gql/Utils';
- import { OnMissing, type FullComicFragment } from '$gql/graphql';
- import { ScrapedComicSelector, getScraperContext } from '$lib/Scraper';
+ import { OnMissing, type FullComicFragment, type ScrapeComicQuery } from '$gql/graphql';
import { toastError, toastFinally } from '$lib/Toasts';
import Select from '$lib/components/Select.svelte';
import Spinner from '$lib/components/Spinner.svelte';
- import { getContextClient } from '@urql/svelte';
+ import { getContextClient, type OperationResult } from '@urql/svelte';
+ import { getScraperContext, ScrapedComicSelector } from './Scraper.svelte';
import SelectorGroup from './components/SelectorGroup.svelte';
import SelectorItem from './components/SelectorItem.svelte';
let client = getContextClient();
const context = getScraperContext();
- export let comic: FullComicFragment;
- let createMissing = false;
- let loading = false;
+ interface Props {
+ comic: FullComicFragment;
+ onupsert: () => void;
+ }
- $: scrapersResult = comicScrapersQuery(client, { id: comic.id });
- $: scrapers = $scrapersResult.data?.comicScrapers;
+ let { comic, onupsert }: Props = $props();
+ let createMissing = $state(false);
+ let loading = $state(false);
- function scrape() {
- loading = true;
- scrapeComic(client, { id: comic.id, scraper: $context.scraper })
- .then((result) => {
- if (result.error) {
- toastError(result.error.message);
- return;
- }
+ let scrapersResult = $derived(comicScrapersQuery(client, { id: comic.id }));
+ let scrapers = $derived($scrapersResult.data?.comicScrapers);
- if (result.data) {
- if (isError(result.data.scrapeComic)) {
- toastError(result.data.scrapeComic.message);
- return;
- }
+ function scrape(event: SubmitEvent) {
+ event.preventDefault();
+ if (!context.scraper) return;
- if (result.data.scrapeComic.__typename === 'ScrapeComicResult') {
- $context.selector = new ScrapedComicSelector(result.data.scrapeComic.data, comic);
- $context.warnings = result.data.scrapeComic.warnings;
- }
- }
- })
+ loading = true;
+ scrapeComic(client, { id: comic.id, scraper: context.scraper })
+ .then(handleScrapeResult)
.catch(toastFinally)
.finally(() => (loading = false));
}
- function updateFromScrape(createMissing: boolean) {
- if (!$context.selector) return;
+ function handleScrapeResult(result: OperationResult<ScrapeComicQuery>) {
+ if (result.error) {
+ toastError(result.error.message);
+ return;
+ }
+
+ if (result.data) {
+ if (isError(result.data.scrapeComic)) {
+ toastError(result.data.scrapeComic.message);
+ return;
+ }
+
+ if (result.data.scrapeComic.__typename === 'ScrapeComicResult') {
+ context.selector = new ScrapedComicSelector(result.data.scrapeComic.data, comic);
+ context.warnings = result.data.scrapeComic.warnings;
+ }
+ }
+ }
+
+ function upsert(event: SubmitEvent) {
+ event.preventDefault();
+ if (!context.selector) return;
- upsertComics(client, {
- ids: comic.id,
- input: $context.selector.toInput(createMissing ? OnMissing.Create : OnMissing.Ignore)
- })
+ const input = context.selector.input(createMissing ? OnMissing.Create : OnMissing.Ignore);
+ upsertComics(client, { ids: comic.id, input })
.then(() => {
- $context.selector = undefined;
- $context.warnings = [];
+ onupsert();
+ context.reset();
})
.catch(toastFinally);
}
@@ -65,56 +74,56 @@
{#if scrapers && scrapers.length === 0}
<h2 class="text-base">No scrapers available.</h2>
{:else}
- <form on:submit|preventDefault={scrape}>
+ <form onsubmit={scrape}>
<div class="grid grid-cols-6 gap-2">
<div class="col-span-5">
<Select
id="scrapers"
options={scrapers}
placeholder={'Select scraper...'}
- bind:value={$context.scraper}
+ bind:value={context.scraper}
/>
</div>
- <button type="submit" disabled={!$context.scraper} class="btn-blue">Scrape</button>
+ <button type="submit" disabled={!context.scraper} class="btn-blue">Scrape</button>
</div>
</form>
{/if}
{#if loading}
<Spinner />
- {:else if $context.selector}
- {#if $context.warnings.length > 0}
+ {:else if context.selector}
+ {#if context.warnings.length > 0}
<div class="flex flex-col gap-2">
<h2 class="flex gap-1 border-b border-slate-700 text-base font-medium">Warnings</h2>
<ul class="ml-2 list-inside list-disc">
- {#each $context.warnings as warning}
+ {#each context.warnings as warning}
<li>{warning}</li>
{/each}
</ul>
</div>
{/if}
- {#if !$context.selector.hasData()}
+ {#if !context.selector.pending()}
<h2 class="text-base">No data to merge.</h2>
{:else}
<div class="flex flex-col gap-2">
<h2 class="border-b border-slate-700 text-base font-medium">Results</h2>
- <form on:submit|preventDefault={() => updateFromScrape(createMissing)}>
+ <form onsubmit={upsert}>
<div class="grid grid-cols-6 gap-4 pb-2">
- <SelectorItem title="Title" selector={$context.selector.title} />
- <SelectorItem title="Original Title" selector={$context.selector.originalTitle} />
- <SelectorItem title="URL" selector={$context.selector.url} />
- <SelectorItem title="Date" selector={$context.selector.date} --span="2" />
- <SelectorItem title="Category" selector={$context.selector.category} --span="2" />
- <SelectorItem title="Language" selector={$context.selector.language} --span="2" />
- <SelectorItem title="Rating" selector={$context.selector.rating} --span="2" />
- <SelectorItem title="Censorship" selector={$context.selector.censorship} --span="2" />
- <SelectorItem title="Direction" selector={$context.selector.direction} --span="2" />
- <SelectorItem title="Layout" selector={$context.selector.layout} --span="2" />
- <SelectorGroup title="Artists" selectors={$context.selector.artists} />
- <SelectorGroup title="Circles" selectors={$context.selector.circles} />
- <SelectorGroup title="Characters" selectors={$context.selector.characters} />
- <SelectorGroup title="Worlds" selectors={$context.selector.worlds} />
- <SelectorGroup title="Tags" selectors={$context.selector.tags} />
+ <SelectorItem title="Title" selector={context.selector.title} />
+ <SelectorItem title="Original Title" selector={context.selector.originalTitle} />
+ <SelectorItem title="URL" selector={context.selector.url} />
+ <SelectorItem title="Date" selector={context.selector.date} --span="2" />
+ <SelectorItem title="Category" selector={context.selector.category} --span="2" />
+ <SelectorItem title="Language" selector={context.selector.language} --span="2" />
+ <SelectorItem title="Rating" selector={context.selector.rating} --span="2" />
+ <SelectorItem title="Censorship" selector={context.selector.censorship} --span="2" />
+ <SelectorItem title="Direction" selector={context.selector.direction} --span="2" />
+ <SelectorItem title="Layout" selector={context.selector.layout} --span="2" />
+ <SelectorGroup title="Artists" selectors={context.selector.artists} />
+ <SelectorGroup title="Circles" selectors={context.selector.circles} />
+ <SelectorGroup title="Characters" selectors={context.selector.characters} />
+ <SelectorGroup title="Worlds" selectors={context.selector.worlds} />
+ <SelectorGroup title="Tags" selectors={context.selector.tags} />
</div>
<div class="flex flex-col gap-2">
<h2 class="border-b border-slate-700 text-base font-medium">Options</h2>
diff --git a/frontend/src/lib/scraper/Scraper.svelte.ts b/frontend/src/lib/scraper/Scraper.svelte.ts
new file mode 100644
index 0000000..93e756b
--- /dev/null
+++ b/frontend/src/lib/scraper/Scraper.svelte.ts
@@ -0,0 +1,164 @@
+import {
+ Category,
+ Censorship,
+ Direction,
+ Language,
+ Layout,
+ OnMissing,
+ Rating,
+ type FullComicFragment,
+ type ScrapedComic,
+ type UpsertComicInput,
+ type UpsertOptions
+} from '$gql/graphql';
+import {
+ CategoryLabel,
+ CensorshipLabel,
+ DirectionLabel,
+ LanguageLabel,
+ LayoutLabel,
+ RatingLabel
+} from '$lib/Enums';
+import { getContext, setContext } from 'svelte';
+
+class ScraperContext {
+ scraper?: string = $state();
+ warnings: string[] = $state([]);
+ selector?: ScrapedComicSelector = $state();
+
+ reset = () => {
+ this.selector = undefined;
+ this.warnings = [];
+ };
+}
+
+export function initScraperContext() {
+ return setContext<ScraperContext>('scraper', new ScraperContext());
+}
+
+export function getScraperContext() {
+ return getContext<ScraperContext>('scraper');
+}
+
+export class Selector<T extends string> {
+ keep = $state(true);
+ value: T;
+ display: string | undefined;
+
+ constructor(value: T, display?: string) {
+ this.value = value;
+ this.display = display;
+ }
+
+ toggle = () => {
+ this.keep = !this.keep;
+ };
+
+ toString() {
+ return this.display ?? this.value;
+ }
+
+ static from<T extends string>(
+ scraped: T | undefined | null,
+ have: string | undefined | null,
+ label?: Record<string, string>
+ ) {
+ if (scraped && have !== scraped) {
+ return new Selector(scraped, label ? label[scraped] : undefined);
+ }
+ return undefined;
+ }
+
+ static fromList(scraped: string[], have: { name: string }[]) {
+ const haves = new Set(have.map((i) => i.name));
+
+ return scraped.filter((i) => !haves.has(i)).map((i) => new Selector(i));
+ }
+}
+
+function keepItem<T extends string>(selector?: Selector<T>): T | undefined | null {
+ if (selector?.keep) {
+ return selector.value;
+ }
+ return undefined;
+}
+
+function keepList<T extends string>(
+ selectorList: Selector<T>[],
+ onMissing: OnMissing
+): { names: T[]; options: UpsertOptions } {
+ return {
+ names: selectorList.filter((v) => v.keep).map((v) => v.value),
+ options: { onMissing }
+ };
+}
+
+export class ScrapedComicSelector {
+ title?: Selector<string>;
+ originalTitle?: Selector<string>;
+ url?: Selector<string>;
+ date?: Selector<string>;
+ category?: Selector<Category>;
+ censorship?: Selector<Censorship>;
+ rating?: Selector<Rating>;
+ language?: Selector<Language>;
+ direction?: Selector<Direction>;
+ layout?: Selector<Layout>;
+ artists: Selector<string>[];
+ circles: Selector<string>[];
+ characters: Selector<string>[];
+ worlds: Selector<string>[];
+ tags: Selector<string>[];
+
+ constructor(scraped: ScrapedComic, comic: FullComicFragment) {
+ this.title = Selector.from(scraped.title, comic.title);
+ this.originalTitle = Selector.from(scraped.originalTitle, comic.originalTitle);
+ this.url = Selector.from(scraped.url, comic.url);
+ this.date = Selector.from(scraped.date, comic.date);
+ this.category = Selector.from(scraped.category, comic.category, CategoryLabel);
+ this.censorship = Selector.from(scraped.censorship, comic.censorship, CensorshipLabel);
+ this.rating = Selector.from(scraped.rating, comic.rating, RatingLabel);
+ this.language = Selector.from(scraped.language, comic.language, LanguageLabel);
+ this.direction = Selector.from(scraped.direction, comic.direction, DirectionLabel);
+ this.layout = Selector.from(scraped.layout, comic.layout, LayoutLabel);
+
+ this.artists = Selector.fromList(scraped.artists, comic.artists);
+ this.circles = Selector.fromList(scraped.circles, comic.circles);
+ this.characters = Selector.fromList(scraped.characters, comic.characters);
+ this.tags = Selector.fromList(scraped.tags, comic.tags);
+ this.worlds = Selector.fromList(scraped.worlds, comic.worlds);
+ }
+
+ pending() {
+ return (
+ Object.values(this).filter((i) => {
+ if (i === undefined) {
+ return false;
+ } else if (Array.isArray(i) && i.length === 0) {
+ return false;
+ }
+ return true;
+ }).length > 0
+ );
+ }
+
+ input(onMissing: OnMissing): UpsertComicInput {
+ return {
+ title: keepItem(this.title),
+ originalTitle: keepItem(this.originalTitle),
+ url: keepItem(this.url),
+ date: keepItem(this.date),
+ category: keepItem(this.category),
+ censorship: keepItem(this.censorship),
+ rating: keepItem(this.rating),
+ language: keepItem(this.language),
+ direction: keepItem(this.direction),
+ layout: keepItem(this.layout),
+ artists: keepList(this.artists, onMissing),
+ circles: keepList(this.circles, onMissing),
+ characters: keepList(this.characters, onMissing),
+ worlds: keepList(this.worlds, onMissing),
+ tags: keepList(this.tags, onMissing)
+ };
+ }
+}
diff --git a/frontend/src/lib/scraper/components/SelectorButton.svelte b/frontend/src/lib/scraper/components/SelectorButton.svelte
index b786f89..e976f91 100644
--- a/frontend/src/lib/scraper/components/SelectorButton.svelte
+++ b/frontend/src/lib/scraper/components/SelectorButton.svelte
@@ -1,19 +1,19 @@
<script lang="ts">
- import { Selector } from '$lib/Scraper';
+ import { Selector } from '../Scraper.svelte';
- export let selector: Selector<string>;
+ let { selector }: { selector: Selector<string> } = $props();
</script>
<button
type="button"
class="ml-1 flex rounded-sm border-slate-700 bg-slate-900 hover:brightness-110"
- on:click={() => (selector.keep = !selector.keep)}
+ onclick={selector.toggle}
>
<div class="flex self-center pl-1">
{#if selector.keep}
- <span class="icon-base icon-[material-symbols--check] text-green-400" />
+ <span class="icon-base icon-[material-symbols--check] text-green-400"></span>
{:else}
- <span class="icon-base icon-[material-symbols--close] text-red-400" />
+ <span class="icon-base icon-[material-symbols--close] text-red-400"></span>
{/if}
</div>
<p class:opacity-50={!selector.keep} class="p-1 text-left">
diff --git a/frontend/src/lib/scraper/components/SelectorGroup.svelte b/frontend/src/lib/scraper/components/SelectorGroup.svelte
index 1fdb8f2..11489b1 100644
--- a/frontend/src/lib/scraper/components/SelectorGroup.svelte
+++ b/frontend/src/lib/scraper/components/SelectorGroup.svelte
@@ -1,9 +1,13 @@
<script lang="ts">
- import { Selector } from '$lib/Scraper';
+ import { Selector } from '../Scraper.svelte';
import SelectorButton from './SelectorButton.svelte';
- export let title: string;
- export let selectors: Selector<string>[];
+ interface Props {
+ title: string;
+ selectors: Selector<string>[];
+ }
+
+ let { title, selectors = $bindable() }: Props = $props();
function invert() {
for (let selector of selectors) {
@@ -20,8 +24,9 @@
<button
type="button"
class="flex items-end opacity-75 brightness-75 transition-opacity hover:opacity-100 hover:brightness-110 focus-visible:opacity-100"
- on:click={invert}
+ onclick={invert}
title="Invert selection"
+ aria-label="Invert selection"
>
<span class="icon-xs icon-[material-symbols--compare-arrows]"></span>
</button>
diff --git a/frontend/src/lib/scraper/components/SelectorItem.svelte b/frontend/src/lib/scraper/components/SelectorItem.svelte
index dd3f5b4..5beba50 100644
--- a/frontend/src/lib/scraper/components/SelectorItem.svelte
+++ b/frontend/src/lib/scraper/components/SelectorItem.svelte
@@ -1,9 +1,8 @@
<script lang="ts">
- import { Selector } from '$lib/Scraper';
+ import { Selector } from '../Scraper.svelte';
import SelectorButton from './SelectorButton.svelte';
- export let title: string;
- export let selector: Selector<string> | undefined;
+ let { title, selector }: { title: string; selector?: Selector<string> } = $props();
</script>
{#if selector}
powered by cgit and git | oriole.systems | how to verify signatures