Skip to content

Commit d4d3671

Browse files
BurnzZBurnzZ
committed
add tests to stat logging feature
1 parent 57f5308 commit d4d3671

File tree

1 file changed

+175
-0
lines changed

1 file changed

+175
-0
lines changed

‎tests/test_loader_stats.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import pytest
2+
from unittest import mock
3+
4+
from itemloaders import ItemLoader
5+
from parsel import Selector
6+
7+
8+
def test_write_to_stats_with_uninjected_stat_dependency():
9+
"""It should not call stats when the stat dependency isn't available."""
10+
11+
loader = ItemLoader()
12+
loader.stats = mock.MagicMock()
13+
loader.stats.__bool__.return_value = False # don't pass the if-condition
14+
15+
assert loader.write_to_stats("field_name", "parsed_data", 0, "xpath") == None
16+
assert not loader.stats.inc_value.called
17+
18+
19+
def test_write_to_stats_with_no_parsed_data():
20+
"""It should not call stats when parsing the data returned None."""
21+
22+
loader = ItemLoader()
23+
loader.stats = mock.Mock()
24+
25+
parsed_data = None
26+
expected_stat_key = "parser/ItemLoader/field_name/css/0/missing"
27+
28+
assert loader.write_to_stats("field_name", parsed_data, 0, "css") == None
29+
loader.stats.inc_value.assert_called_once_with(expected_stat_key)
30+
31+
32+
def test_write_to_stats_with_no_field_name():
33+
"""It should not call stats when the 'field_name' passed is None."""
34+
35+
loader = ItemLoader()
36+
loader.stats = mock.Mock()
37+
38+
assert loader.write_to_stats(None, "sample data", 0, "css") == None
39+
loader.stats.inc_value.assert_not_called()
40+
41+
42+
def test_write_to_stats():
43+
"""It should incremenent the correct key in the stat."""
44+
45+
loader = ItemLoader()
46+
loader.stats = mock.MagicMock()
47+
48+
expected_stat_key = "parser/ItemLoader/field_name/css/0"
49+
50+
# Rules with values
51+
assert loader.write_to_stats("field_name", "parsed_data", 123, "css") == None
52+
53+
# Rules that hasn't rendered any values
54+
assert loader.write_to_stats("field_name", None, 456, "css") == None
55+
assert loader.write_to_stats("field_name", [], 789, "css") == None
56+
57+
loader.stats.inc_value.assert_has_calls(
58+
[
59+
mock.call("parser/ItemLoader/field_name/css/123"),
60+
mock.call("parser/ItemLoader/field_name/css/456/missing"),
61+
mock.call("parser/ItemLoader/field_name/css/789/missing"),
62+
]
63+
)
64+
65+
66+
TEST_HTML_BODY = """
67+
<html>
68+
<title>This is a title</title>
69+
<body>
70+
<article>
71+
<h2>Product #1</h2>
72+
<span class='price'>$1.23</span>
73+
</article>
74+
75+
<article>
76+
<div class='product-title'>Product #2</div>
77+
<span class='price'>$9.99</span>
78+
</article>
79+
</body>
80+
</html>
81+
"""
82+
83+
84+
class TestItemLoader(ItemLoader):
85+
pass
86+
87+
88+
@pytest.fixture()
89+
def loader():
90+
mock_stats = mock.MagicMock()
91+
selector = Selector(text=TEST_HTML_BODY)
92+
loader = TestItemLoader(selector=selector, stats=mock_stats)
93+
return loader
94+
95+
96+
# NOTES: We'll be using the 'css' methods of ItemLoader below. The 'xpath'
97+
# methods are also using the 'get_selector_values()' method underneath, the
98+
# same with 'css'. So we'll assume that 'xpath' would also pass the test
99+
# if 'css' passes.
100+
101+
# This assumption will hold true for now, since the current implementation of
102+
# the 'css' and 'xpath' methods are just facades to the 'get_selector_values()'.
103+
104+
105+
def test_add_css_1(loader):
106+
loader.add_css("title", "article h2::text")
107+
loader.stats.inc_value.assert_has_calls(
108+
[mock.call("parser/TestItemLoader/title/css/1")]
109+
)
110+
assert loader.stats.inc_value.call_count == 1
111+
112+
113+
def test_add_css_2(loader):
114+
loader.add_css("title", ["article h2::text", "article .product-title::text"])
115+
loader.stats.inc_value.assert_has_calls(
116+
[
117+
mock.call("parser/TestItemLoader/title/css/1"),
118+
mock.call("parser/TestItemLoader/title/css/2"),
119+
]
120+
)
121+
assert loader.stats.inc_value.call_count == 2
122+
123+
124+
def test_add_css_3_missing(loader):
125+
loader.add_css("title", "h1::text") # The <h1> doesn't exist at all.
126+
loader.stats.inc_value.assert_has_calls(
127+
[mock.call("parser/TestItemLoader/title/css/1/missing")]
128+
)
129+
assert loader.stats.inc_value.call_count == 1
130+
131+
132+
def test_multiple_1(loader):
133+
loader.add_css("title", "h2::text")
134+
loader.add_css("title", ["article h2::text", "article .product-title::text"])
135+
loader.stats.inc_value.assert_has_calls(
136+
[
137+
mock.call("parser/TestItemLoader/title/css/1"),
138+
mock.call("parser/TestItemLoader/title/css/2"),
139+
mock.call("parser/TestItemLoader/title/css/3"),
140+
]
141+
)
142+
assert loader.stats.inc_value.call_count == 3
143+
144+
145+
def test_multiple_1_with_name(loader):
146+
loader.add_css("title", "h2::text", name="title from h2")
147+
loader.add_css(
148+
"title",
149+
["article h2::text", "article .product-title::text"],
150+
name="title from article",
151+
)
152+
loader.stats.inc_value.assert_has_calls(
153+
[
154+
mock.call("parser/TestItemLoader/title/css/1/title from h2"),
155+
mock.call("parser/TestItemLoader/title/css/2/title from article"),
156+
mock.call("parser/TestItemLoader/title/css/3/title from article"),
157+
]
158+
)
159+
assert loader.stats.inc_value.call_count == 3
160+
161+
162+
def test_multiple_2_with_name(loader):
163+
loader.add_css("title", "h2::text", name="title from h2")
164+
loader.add_xpath("title", "//article/h2/text()", name="title from article")
165+
loader.add_css("title", "article .product-title::text")
166+
loader.add_xpath("title", "//aside/h1/text()", name="title from aside")
167+
loader.stats.inc_value.assert_has_calls(
168+
[
169+
mock.call("parser/TestItemLoader/title/css/1/title from h2"),
170+
mock.call("parser/TestItemLoader/title/xpath/1/title from article"),
171+
mock.call("parser/TestItemLoader/title/css/2"),
172+
mock.call("parser/TestItemLoader/title/xpath/2/title from aside/missing"),
173+
]
174+
)
175+
assert loader.stats.inc_value.call_count == 4

0 commit comments

Comments
 (0)