feat: support wider variety of spider names

Author: honzajavorek

src/apify/scrapy/extensions/_httpcache.py

@@ -3,6 +3,7 @@
 import gzip
 import io
 import pickle
+import re
 import struct
 from logging import getLogger
 from time import time
@@ -47,7 +48,7 @@ def open_spider(self, spider: Spider) -> None:
         logger.debug('Using Apify key value cache storage', extra={'spider': spider})
         self._spider = spider
         self._fingerprinter = spider.crawler.request_fingerprinter
-        kvs_name = f'httpcache-{spider.name}'
+        kvs_name = get_kvs_name(spider.name)
 
         async def open_kvs() -> KeyValueStore:
             config = Configuration.get_global_configuration()
@@ -177,3 +178,13 @@ def read_gzip_time(gzip_bytes: bytes) -> int:
     header_components = struct.unpack('<HBBI2B', header)
     mtime: int = header_components[3]
     return mtime
+
+
+def get_kvs_name(spider_name: str) -> str:
+    """Get the key value store name for a spider."""
+    slug = re.sub(r'[^a-zA-Z0-9-]', '-', spider_name)
+    slug = re.sub(r'-+', '-', slug)
+    slug = slug.strip('-')
+    if not slug:
+        raise ValueError(f'Unsupported spider name: {spider_name!r} (slug: {slug!r})')
+    return f'httpcache-{slug}'

tests/unit/scrapy/extensions/test_httpcache.py

@@ -1,6 +1,8 @@
 from time import time
 
-from apify.scrapy.extensions._httpcache import from_gzip, read_gzip_time, to_gzip
+import pytest
+
+from apify.scrapy.extensions._httpcache import from_gzip, get_kvs_name, read_gzip_time, to_gzip
 
 FIXTURE_DICT = {'name': 'Alice'}
 
@@ -35,3 +37,34 @@ def test_read_gzip_time_non_zero() -> None:
     data_bytes = to_gzip(FIXTURE_DICT, mtime=current_time)
 
     assert read_gzip_time(data_bytes) == current_time
+
+
+@pytest.mark.parametrize(
+    ('spider_name', 'expected'),
+    [
+        ('test', 'httpcache-test'),
+        ('123', 'httpcache-123'),
+        ('test-spider', 'httpcache-test-spider'),
+        ('test_spider', 'httpcache-test-spider'),
+        ('test spider', 'httpcache-test-spider'),
+        ('test👻spider', 'httpcache-test-spider'),
+        ('test@spider', 'httpcache-test-spider'),
+        ('   test   spider   ', 'httpcache-test-spider'),
+        ('testspider.com', 'httpcache-testspider-com'),
+    ],
+)
+def test_get_kvs_name(spider_name: str, expected: str) -> None:
+    assert get_kvs_name(spider_name) == expected
+
+
+@pytest.mark.parametrize(
+    ('spider_name'),
+    [
+        '',
+        '-',
+        '-@-/-',
+    ],
+)
+def test_get_kvs_name_raises(spider_name: str) -> None:
+    with pytest.raises(ValueError, match='Unsupported spider name'):
+        assert get_kvs_name(spider_name)