initial migration of Burnzz/scrapy-loader-upkeep code

Author: BurnzZ

itemloaders/__init__.py

@@ -86,7 +86,7 @@ class ItemLoader:
     default_input_processor = Identity()
     default_output_processor = Identity()
 
-    def __init__(self, item=None, selector=None, parent=None, **context):
+    def __init__(self, item=None, selector=None, parent=None, stats=None, **context):
         self.selector = selector
         context.update(selector=selector)
         if item is None:
@@ -99,6 +99,14 @@ def __init__(self, item=None, selector=None, parent=None, **context):
         for field_name, value in item.items():
             self._values[field_name] += arg_to_iter(value)
 
+        # This is the new injected dependency that we'll be using as the main
+        # functionality of this tool.
+        self.stats = stats
+
+        # This keeps track of the position of the 'field' name that is being
+        # loaded for a more accurate logging in the stats.
+        self.field_tracker = defaultdict(int)
+
     @property
     def _values(self):
         if self.parent is not None:
@@ -327,14 +335,15 @@ def add_xpath(self, field_name, xpath, *processors, **kw):
             loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
 
         """
-        values = self._get_xpathvalues(xpath, **kw)
+        self.field_tracker[f"{field_name}_xpath"] += 1
+        values = self._get_xpathvalues(field_name, xpath, **kw)
         self.add_value(field_name, values, *processors, **kw)
 
     def replace_xpath(self, field_name, xpath, *processors, **kw):
         """
         Similar to :meth:`add_xpath` but replaces collected data instead of adding it.
         """
-        values = self._get_xpathvalues(xpath, **kw)
+        values = self._get_xpathvalues(field_name, xpath, **kw)
         self.replace_value(field_name, values, *processors, **kw)
 
     def get_xpath(self, xpath, *processors, **kw):
@@ -358,13 +367,11 @@ def get_xpath(self, xpath, *processors, **kw):
             loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
 
         """
-        values = self._get_xpathvalues(xpath, **kw)
+        values = self._get_xpathvalues(None, xpath, **kw)
         return self.get_value(values, *processors, **kw)
 
-    def _get_xpathvalues(self, xpaths, **kw):
-        self._check_selector_method()
-        xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
+    def _get_xpathvalues(self, field_name, xpaths, **kw):
+        return self.get_selector_values(field_name, xpaths, self.selector.xpath, **kw)
 
     def add_css(self, field_name, css, *processors, **kw):
         """
@@ -384,14 +391,15 @@ def add_css(self, field_name, css, *processors, **kw):
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.add_css('price', 'p#price', re='the price is (.*)')
         """
-        values = self._get_cssvalues(css, **kw)
+        self.field_tracker[f"{field_name}_css"] += 1
+        values = self._get_cssvalues(field_name, css, **kw)
         self.add_value(field_name, values, *processors, **kw)
 
     def replace_css(self, field_name, css, *processors, **kw):
         """
         Similar to :meth:`add_css` but replaces collected data instead of adding it.
         """
-        values = self._get_cssvalues(css, **kw)
+        values = self._get_cssvalues(field_name, css, **kw)
         self.replace_value(field_name, values, *processors, **kw)
 
     def get_css(self, css, *processors, **kw):
@@ -414,10 +422,68 @@ def get_css(self, css, *processors, **kw):
             # HTML snippet: <p id="price">the price is $1200</p>
             loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
         """
-        values = self._get_cssvalues(css, **kw)
+        values = self._get_cssvalues(None, css, **kw)
         return self.get_value(values, *processors, **kw)
 
-    def _get_cssvalues(self, csss, **kw):
+    def _get_cssvalues(self, field_name, csss, **kw):
+        return self.get_selector_values(field_name, csss, self.selector.css, **kw)
+
+    def get_selector_values(self, field_name, selector_rules, selector, **kw):
+        """Provides an abstraction to _get_xpathvalues() and _get_cssvalues()
+        since they share the same components.
+        """
+
         self._check_selector_method()
-        csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).getall() for css in csss)
+
+        selector_type = selector.__name__  # either 'css' or 'xpath'
+
+        # The optional arg in methods like `add_css()` for context in stats
+        name = kw.get("name")
+
+        # For every call of `add_css()` and `add_xpath()` this is incremented.
+        # We'll use it as the base index of the position of the logged stats.
+        index = self.field_tracker[f"{field_name}_{selector_type}"]
+
+        values = []
+        for position, rule in enumerate(arg_to_iter(selector_rules), index):
+            parsed_data = selector(rule).getall()
+            values.append(parsed_data)
+            self.write_to_stats(
+                field_name, parsed_data, position, selector_type, name=name
+            )
+        return flatten(values)
+
+    def write_to_stats(
+        self, field_name, parsed_data, position, selector_type, name=None
+    ):
+        """Responsible for logging the parser rules usage.
+
+        NOTES: It's hard to easily denote which parser rule hasn't produced any
+          data for the entire crawl, since ItemLoaders essentially don't know
+          when the spider is going to be closed, as well as it has many
+          instantiations all throughout the code.
+
+        The implementation below where each missing parsed_data is being logged
+        to the stat is clunky, but necessary. With this, we can only surmise
+        that it's safe to remove parser fallback parser if it's all just
+        '*/missing' in the stats.
+        """
+
+        if not self.stats or not field_name:
+            return
+
+        parser_label = (
+            f"parser/{self.loader_name}/{field_name}/{selector_type}/{position}"
+        )
+
+        if name:
+            parser_label += f"/{name}"
+
+        if parsed_data in (None, []):
+            parser_label += "/missing"
+
+        self.stats.inc_value(parser_label)
+
+    @property
+    def loader_name(self):
+        return self.__class__.__name__