diff options
| -rw-r--r-- | src/hircine/scraper/__init__.py | 9 | ||||
| -rw-r--r-- | tests/scrapers/test_scraper.py | 15 |
2 files changed, 21 insertions, 3 deletions
diff --git a/src/hircine/scraper/__init__.py b/src/hircine/scraper/__init__.py index bc83436..4606f4a 100644 --- a/src/hircine/scraper/__init__.py +++ b/src/hircine/scraper/__init__.py @@ -86,13 +86,16 @@ class Scraper(ABC): transformers = [] def generator(): - for result in self.scrape(): - if callable(result): + for item in self.scrape(): + if callable(item): try: - yield result() + result = item() except ScrapeWarning as e: self.log_warning(e) else: + result = item + + if result is not None: yield result gen = generator() diff --git a/tests/scrapers/test_scraper.py b/tests/scrapers/test_scraper.py index 6f6f29d..8492425 100644 --- a/tests/scrapers/test_scraper.py +++ b/tests/scrapers/test_scraper.py @@ -9,6 +9,14 @@ class MockScraper(Scraper): yield "bar" +class NoneScraper(Scraper): + is_available = True + + def scrape(self): + yield lambda: "foo" + yield None + + class WarningScraper(Scraper): is_available = True @@ -53,3 +61,10 @@ def test_scraper_collects_literal(): generator = scraper.collect() assert set(generator) == set(["literal"]) + + +def test_scraper_collect_ignores_none(): + scraper = NoneScraper(None) + generator = scraper.collect() + + assert set(generator) == set(["foo"]) |