Merge branch 'multi-exec-on-cluster' of github.com:robertosantamaria-scopely/redis-py into multi-exec-on-cluster

Author: vladvildanov

.github/actions/run-tests/action.yml

@@ -14,6 +14,10 @@ inputs:
     description: 'hiredis version to test against'
     required: false
     default: '>3.0.0'
+  hiredis-branch:
+    description: 'hiredis branch to test against'
+    required: false
+    default: 'master'
   event-loop:
     description: 'Event loop to use'
     required: false
@@ -28,6 +32,14 @@ runs:
         python-version: ${{ inputs.python-version }}
         cache: 'pip'
 
+    - uses: actions/checkout@v4
+      if: ${{ inputs.parser-backend == 'hiredis' && inputs.hiredis-version == 'unstable' }}
+      with:
+        repository: redis/hiredis-py
+        submodules: true
+        path: hiredis-py
+        ref: ${{ inputs.hiredis-branch }}
+
     - name: Setup Test environment
       env:
         REDIS_VERSION: ${{ inputs.redis-version }}
@@ -40,15 +52,21 @@ runs:
         pip uninstall -y redis  # uninstall Redis package installed via redis-entraid
         pip install -e .[jwt]  # install the working copy
         if [ "${{inputs.parser-backend}}" == "hiredis" ]; then
-          pip install "hiredis${{inputs.hiredis-version}}"
-          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV        
+          if [[ "${{inputs.hiredis-version}}" == "unstable" ]]; then
+            echo "Installing unstable version of hiredis from local directory"
+            pip install -e ./hiredis-py
+          else
+            pip install "hiredis${{inputs.hiredis-version}}"
+          fi          
+          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV
         else
           echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV
         fi
         echo "::endgroup::"
         
         echo "::group::Starting Redis servers"
         redis_major_version=$(echo "$REDIS_VERSION" | grep -oP '^\d+')
+        echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV
         
         if (( redis_major_version < 8 )); then
           echo "Using redis-stack for module tests"
@@ -70,8 +88,7 @@ runs:
                   
           if (( redis_major_version < 7 )); then
             export REDIS_STACK_EXTRA_ARGS="--tls-auth-clients optional --save ''"
-            export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''"
-            echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV
+            export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''"            
           fi
         
           invoke devenv --endpoints=all-stack
@@ -108,12 +125,10 @@ runs:
           fi
 
           echo "::endgroup::"
-    
-          if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then
-            echo "::group::RESP${protocol} cluster tests"
-            invoke cluster-tests $eventloop --protocol=${protocol}
-            echo "::endgroup::"
-          fi
+            
+          echo "::group::RESP${protocol} cluster tests"
+          invoke cluster-tests $eventloop --protocol=${protocol}
+          echo "::endgroup::"          
         }
 
         run_tests 2 "${{inputs.event-loop}}"

.github/workflows/hiredis-py-integration.yaml

@@ -0,0 +1,66 @@
+name: Hiredis-py integration tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      redis-py-branch:
+        description: 'redis-py branch to run tests on'
+        required: true
+        default: 'master'
+      hiredis-branch:
+        description: 'hiredis-py branch to run tests on'
+        required: true
+        default: 'master'
+
+concurrency:
+  group: ${{ github.event.pull_request.number || github.ref }}-hiredis-integration
+  cancel-in-progress: true
+
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+  # this speeds up coverage with Python 3.12: https://github.com/nedbat/coveragepy/issues/1665
+  COVERAGE_CORE: sysmon
+  CURRENT_CLIENT_LIBS_TEST_STACK_IMAGE_TAG: 'rs-7.4.0-v2'
+  CURRENT_REDIS_VERSION: '7.4.2'
+
+jobs:
+  redis_version:
+    runs-on: ubuntu-latest
+    outputs:
+      CURRENT: ${{ env.CURRENT_REDIS_VERSION }}
+    steps:
+      - name: Compute outputs
+        run: |
+          echo "CURRENT=${{ env.CURRENT_REDIS_VERSION }}" >> $GITHUB_OUTPUT  
+
+  hiredis-tests:
+    runs-on: ubuntu-latest
+    needs: [redis_version]
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 15
+      fail-fast: false
+      matrix:
+        redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ]
+        python-version: [ '3.8', '3.13']
+        parser-backend: [ 'hiredis' ]
+        hiredis-version: [ 'unstable' ]
+        event-loop: [ 'asyncio' ]
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}} (${{ matrix.hiredis-version }}); EL:${{matrix.event-loop}}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.redis-py-branch }}
+      - name: Run tests
+        uses: ./.github/actions/run-tests
+        with:
+          python-version: ${{ matrix.python-version }}
+          parser-backend: ${{ matrix.parser-backend }}
+          redis-version: ${{ matrix.redis-version }}
+          hiredis-version: ${{ matrix.hiredis-version }}
+          hiredis-branch: ${{ inputs.hiredis-branch }}

.github/workflows/integration.yaml

@@ -74,7 +74,7 @@ jobs:
       max-parallel: 15
       fail-fast: false
       matrix:
-        redis-version: ['8.0-RC2-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.7', '6.2.17']
+        redis-version: ['8.0.1-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.7', '6.2.17']
         python-version: ['3.8', '3.13']
         parser-backend: ['plain']
         event-loop: ['asyncio']

CHANGES

@@ -71,6 +71,7 @@
     * Close Unix sockets if the connection attempt fails. This prevents `ResourceWarning`s. (#3314)
     * Close SSL sockets if the connection attempt fails, or if validations fail. (#3317)
     * Eliminate mutable default arguments in the `redis.commands.core.Script` class. (#3332)
+    * Allow newer versions of PyJWT as dependency. (#3630)
 
 * 4.1.3 (Feb 8, 2022)
   * Fix flushdb and flushall (#1926)

pyproject.toml

@@ -41,7 +41,7 @@ ocsp = [
     "requests>=2.31.0",
 ]
 jwt = [
-    "PyJWT~=2.9.0",
+    "PyJWT>=2.9.0",
 ]
 
 [project.urls]

redis/asyncio/cluster.py

@@ -1313,7 +1313,9 @@ async def initialize(self) -> None:
         startup_nodes_reachable = False
         fully_covered = False
         exception = None
-        for startup_node in self.startup_nodes.values():
+        # Convert to tuple to prevent RuntimeError if self.startup_nodes
+        # is modified during iteration
+        for startup_node in tuple(self.startup_nodes.values()):
             try:
                 # Make sure cluster mode is enabled on this node
                 try:

redis/cluster.py

@@ -1703,7 +1703,9 @@ def initialize(self):
         fully_covered = False
         kwargs = self.connection_kwargs
         exception = None
-        for startup_node in self.startup_nodes.values():
+        # Convert to tuple to prevent RuntimeError if self.startup_nodes
+        # is modified during iteration
+        for startup_node in tuple(self.startup_nodes.values()):
             try:
                 if startup_node.redis_connection:
                     r = startup_node.redis_connection

redis/commands/redismodules.py

@@ -1,28 +1,38 @@
+from __future__ import annotations
+
 from json import JSONDecoder, JSONEncoder
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .bf import BFBloom, CFBloom, CMSBloom, TDigestBloom, TOPKBloom
+    from .json import JSON
+    from .search import AsyncSearch, Search
+    from .timeseries import TimeSeries
+    from .vectorset import VectorSet
 
 
 class RedisModuleCommands:
     """This class contains the wrapper functions to bring supported redis
     modules into the command namespace.
     """
 
-    def json(self, encoder=JSONEncoder(), decoder=JSONDecoder()):
+    def json(self, encoder=JSONEncoder(), decoder=JSONDecoder()) -> JSON:
         """Access the json namespace, providing support for redis json."""
 
         from .json import JSON
 
         jj = JSON(client=self, encoder=encoder, decoder=decoder)
         return jj
 
-    def ft(self, index_name="idx"):
+    def ft(self, index_name="idx") -> Search:
         """Access the search namespace, providing support for redis search."""
 
         from .search import Search
 
         s = Search(client=self, index_name=index_name)
         return s
 
-    def ts(self):
+    def ts(self) -> TimeSeries:
         """Access the timeseries namespace, providing support for
         redis timeseries data.
         """
@@ -32,47 +42,47 @@ def ts(self):
         s = TimeSeries(client=self)
         return s
 
-    def bf(self):
+    def bf(self) -> BFBloom:
         """Access the bloom namespace."""
 
         from .bf import BFBloom
 
         bf = BFBloom(client=self)
         return bf
 
-    def cf(self):
+    def cf(self) -> CFBloom:
         """Access the bloom namespace."""
 
         from .bf import CFBloom
 
         cf = CFBloom(client=self)
         return cf
 
-    def cms(self):
+    def cms(self) -> CMSBloom:
         """Access the bloom namespace."""
 
         from .bf import CMSBloom
 
         cms = CMSBloom(client=self)
         return cms
 
-    def topk(self):
+    def topk(self) -> TOPKBloom:
         """Access the bloom namespace."""
 
         from .bf import TOPKBloom
 
         topk = TOPKBloom(client=self)
         return topk
 
-    def tdigest(self):
+    def tdigest(self) -> TDigestBloom:
         """Access the bloom namespace."""
 
         from .bf import TDigestBloom
 
         tdigest = TDigestBloom(client=self)
         return tdigest
 
-    def vset(self):
+    def vset(self) -> VectorSet:
         """Access the VectorSet commands namespace."""
 
         from .vectorset import VectorSet
@@ -82,7 +92,7 @@ def vset(self):
 
 
 class AsyncRedisModuleCommands(RedisModuleCommands):
-    def ft(self, index_name="idx"):
+    def ft(self, index_name="idx") -> AsyncSearch:
         """Access the search namespace, providing support for redis search."""
 
         from .search import AsyncSearch