Merge branch 'main' into feat/warc-by-cdx

Author: malteos

.github/workflows/ci-slow.yaml

@@ -16,7 +16,7 @@ jobs:
           - python-version: '3.12'
             os: macos-latest
             EXTRA: true
-          - python-version: '3.7'
+          - python-version: '3.8'
             os: windows-latest
             EXTRA: true
           - python-version: '3.12'

.github/workflows/ci.yaml

@@ -1,34 +1,38 @@
 name: CI
 
-on: workflow_dispatch
-# Disabled for this feature
-# on:
-#   # runtime is erratic and up to an hour
-#   push:
-#     branches:
-#     - main
-#   pull_request:
-#     branches:
-#     - main
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
 
 jobs:
   unit-tests:
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
-      max-parallel: 1  # avoids ever triggering a rate limit
+      fail-fast: true
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: [
+          '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'
+        ]
         os: [ubuntu-latest]
         EXTRA: [false]  # used to force includes to get included
         include:
-          - python-version: '3.12'
-            os: ubuntu-latest
+          - python-version: '3.8'
+            os: ubuntu-22.04  # oldest version on github actions
             EXTRA: true
+          - python-version: '3.13'
+            os: ubuntu-latest
             env:
               LOGLEVEL=DEBUG
-          - python-version: '3.7'
-            os: ubuntu-20.04  # oldest version on github actions
+            EXTRA: true
+          - python-version: '3.13'
+            os: macos-latest
+            EXTRA: true
+          - python-version: '3.13'
+            os: windows-latest
             EXTRA: true
 
     steps:
@@ -40,7 +44,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Get Runner IP
+        run: |
+          echo "Runner IP: $(curl -s https://ipinfo.io/ip)"
+
       - name: Install setuptools on python 3.12+
+        # apparently this is a quirk of Github action runners?
         if: ${{ matrix.python-version >= '3.12' }}
         run: |
           pip install setuptools

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 cdx_toolkit.egg-info
 .coverage
 .eggs/
+tmp/

cdx_toolkit/myrequests.py

@@ -54,7 +54,16 @@ def update_next_fetch(hostname, next_fetch):
     retry_info[hostname]['next_fetch'] = next_fetch
 
 
-def myrequests_get(url, params=None, headers=None, cdx=False, allow404=False):
+def myrequests_get(
+    url, 
+    params=None, 
+    headers=None, 
+    cdx=False, 
+    allow404=False, 
+    raise_error_after_n_errors: int = 100, 
+    raise_warning_after_n_errors: int = 10,
+    retry_max_sec: int = 60,
+    ):
     t = time.time()
 
     hostname = urlparse(url).hostname
@@ -84,7 +93,6 @@ def myrequests_get(url, params=None, headers=None, cdx=False, allow404=False):
 
     retry = True
     retry_sec = 2 * minimum_interval
-    retry_max_sec = 60
     retries = 0
     connect_errors = 0
     while retry:
@@ -125,14 +133,17 @@ def myrequests_get(url, params=None, headers=None, cdx=False, allow404=False):
             connect_errors += 1
             string = '{} failures for url {} {!r}: {}'.format(connect_errors, url, params, str(e))
 
-            if 'Name or service not known' in string:
+            # Check for DNS errors with different operating systems
+            if (('Name or service not known' in string)  # linux
+                or ('nodename nor servname provided, or not known' in string)  # macos
+                or ('getaddrinfo failed' in string)):  # windows
                 if dns_fatal(url):
                     raise ValueError('invalid hostname in url '+url) from None
 
-            if connect_errors > 100:
+            if connect_errors > raise_error_after_n_errors:
                 LOGGER.error(string)
                 raise ValueError(string)
-            if connect_errors > 10:
+            if connect_errors > raise_warning_after_n_errors:
                 LOGGER.warning(string)
             LOGGER.info('retrying after {:.2f}s for '.format(retry_max_sec)+str(e))
             time.sleep(retry_max_sec)  # notice the extra-long sleep

requirements.txt

@@ -13,6 +13,7 @@ pytest-cov==2.12.1
 pytest-sugar==0.9.4
 coveralls==3.1.0
 botocore>=1.39.11
+responses==0.25.8
 
 # packaging
 twine==3.4.1

setup.py

@@ -10,7 +10,7 @@
 # remember: keep requires synchronized with requirements.txt
 requires = ['requests', 'warcio', 'fsspec[s3]', 'aioboto3', 'surt', 'tqdm']
 
-test_requirements = ['pytest', 'pytest-cov', 'boto3']
+test_requirements = ['pytest', 'pytest-cov', 'responses']
 
 package_requirements = ['twine', 'setuptools', 'setuptools-scm']
 
@@ -35,7 +35,7 @@
     author_email='lindahl@pbm.com',
     url='https://github.com/cocrawler/cdx_toolkit',
     packages=packages,
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     extras_require=extras_require,
     setup_requires=['setuptools-scm'],
     install_requires=requires,
@@ -58,12 +58,13 @@
         'Programming Language :: Python',
         #'Programming Language :: Python :: 3.5',  # setuptools-scm problem
         #'Programming Language :: Python :: 3.6',  # not offered in github actions
-        'Programming Language :: Python :: 3.7',
+        # 'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'Programming Language :: Python :: 3 :: Only',
     ],
 )