Save attributes from SeleniumRequest into meta

Author: laggardkernel

scrapy_selenium/http.py

@@ -6,7 +6,17 @@
 class SeleniumRequest(Request):
     """Scrapy ``Request`` subclass providing additional arguments"""
 
-    def __init__(self, wait_time=None, wait_until=None, screenshot=False, script=None, *args, **kwargs):
+    var_names = ["wait_time", "wait_until", "screenshot", "script"]
+
+    def __init__(
+        self,
+        wait_time=None,
+        wait_until=None,
+        screenshot=False,
+        script=None,
+        *args,
+        **kwargs,
+    ):
         """Initialize a new selenium request
 
         Parameters
@@ -30,3 +40,14 @@ def __init__(self, wait_time=None, wait_until=None, screenshot=False, script=Non
         self.script = script
 
         super().__init__(*args, **kwargs)
+
+        for name, value in zip(
+            self.var_names, map(lambda x: getattr(self, x), self.var_names)
+        ):
+            if value:
+                self.meta[name] = value
+
+    def unpack_meta(self):
+        for name in self.var_names:
+            setattr(self, name, self.meta.get(name))
+        return self

scrapy_selenium/middlewares.py

@@ -100,6 +100,8 @@ def process_request(self, request, spider):
         if not isinstance(request, SeleniumRequest):
             return None
 
+        request.unpack_meta()
+
         self.driver.get(request.url)
 
         for cookie_name, cookie_value in request.cookies.items():
@@ -137,4 +139,3 @@ def spider_closed(self):
         """Shutdown the driver when spider is closed"""
 
         self.driver.quit()
-