From 47d464fbfc1dd4174c4f0ab39268297c14b972a3 Mon Sep 17 00:00:00 2001
From: Wolfgang Müller
Date: Thu, 14 Nov 2024 22:05:05 +0100
Subject: backend/scraper: Have collect() ignore None results

If a parser function returned None we yield it regardless, even though
it won't have any impact further down the line. Instead clean up the
collect() stream as early as possible.
---
 src/hircine/scraper/__init__.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/hircine/scraper/__init__.py b/src/hircine/scraper/__init__.py
index bc83436..4606f4a 100644
--- a/src/hircine/scraper/__init__.py
+++ b/src/hircine/scraper/__init__.py
@@ -86,13 +86,16 @@ class Scraper(ABC):
             transformers = []
 
         def generator():
-            for result in self.scrape():
-                if callable(result):
+            for item in self.scrape():
+                if callable(item):
                     try:
-                        yield result()
+                        result = item()
                     except ScrapeWarning as e:
                         self.log_warning(e)
                 else:
+                    result = item
+
+                if result is not None:
                     yield result
 
         gen = generator()
-- 
cgit v1.2.3-2-gb3c3