blob: d0cef7b7b457edd08cb9c9ab13ad479b850a14e6 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
from hircine.scraper import Scraper, ScrapeWarning
class MockScraper(Scraper):
is_available = True
def scrape(self):
yield lambda: "foo"
yield "bar"
class NoneScraper(Scraper):
is_available = True
def scrape(self):
yield lambda: "foo"
yield None
class WarningScraper(Scraper):
is_available = True
def warn(self, msg):
raise ScrapeWarning("Invalid input")
def scrape(self):
yield lambda: "foo"
yield lambda: self.warn("bar")
yield "baz"
class ParserlessScraper(Scraper):
is_available = True
def scrape(self):
yield "literal"
def test_scraper_collects():
generator = MockScraper(None).collect()
assert set(generator) == set(["foo", "bar"])
def test_scraper_collects_with_transformer():
generator = MockScraper(None).collect([lambda gen, info: map(str.upper, gen)])
assert set(generator) == set(["FOO", "BAR"])
def test_scraper_collects_warnings():
scraper = WarningScraper(None)
generator = scraper.collect()
assert set(generator) == set(["foo", "baz"])
assert scraper.get_warnings() == ["Invalid input"]
def test_scraper_collects_literal():
scraper = ParserlessScraper(None)
generator = scraper.collect()
assert set(generator) == set(["literal"])
def test_scraper_collect_ignores_none():
scraper = NoneScraper(None)
generator = scraper.collect()
assert set(generator) == set(["foo"])
|
