fix: Update estela-entrypoint to handle scrapy >2.10 versions (#34)

fix: Update estela-entrypoint to handle scrapy >2.10 versions

Author: Adriana618

estela_scrapy/extensions.py

@@ -1,6 +1,6 @@
 import json
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 
 import redis
 from scrapy import signals
@@ -32,7 +32,7 @@ class ItemStorageExtension(BaseExtension):
     def __init__(self, stats):
         super().__init__(stats)
         exporter_kwargs = {"binary": False}
-        self.exporter = PythonItemExporter(**exporter_kwargs)
+        self.exporter = PythonItemExporter(**exporter_kwargs, dont_fail=True)
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -109,8 +109,12 @@ def spider_closed(self, spider, reason):
 
     def store_stats(self, spider):
         stats = self.stats.get_stats()
-        elapsed_time = int((datetime.now() - stats.get("start_time")).total_seconds())
-        stats.update({"elapsed_time_seconds": elapsed_time})
-
+        start_time = stats.get("start_time")
+    
+        if start_time is not None:
+            now = datetime.now(timezone.utc) if start_time.tzinfo else datetime.now()
+            elapsed_time = int((now - start_time).total_seconds())
+            stats.update({"elapsed_time_seconds": elapsed_time})
+    
         parsed_stats = json.dumps(stats, default=json_serializer)
         self.redis_conn.hmset(self.stats_key, json.loads(parsed_stats))

estela_scrapy/middlewares.py

@@ -4,7 +4,7 @@
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_bytes
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import fingerprint as request_fingerprint
 from twisted.web import http
 
 from estela_scrapy.utils import parse_time, producer