Speed up cluster slot initialization

Speed up the computation for slots when initializing a cluster. After
profiling, this turned out to be very slow, when it does not have to be.

Do profiling a bit differently, via pytest.

Author: gerzse

.github/workflows/integration.yaml

@@ -54,7 +54,7 @@ jobs:
           pip install -r dev_requirements.txt
           invoke linters
 
-  run-tests:
+  resp2-tests:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -66,7 +66,7 @@ jobs:
         connection-type: ['hiredis', 'plain']
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-    name: Python ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}} tests
+    name: RESP2 ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}}
     steps:
       - uses: actions/checkout@v4
 
@@ -94,7 +94,8 @@ jobs:
           name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}}
           path: |
             ${{matrix.test-type}}*-results.xml
-            ${{matrix.test-type}}-profile.out
+            prof/**
+            profile_output*
           if-no-files-found: error
           retention-days: 10
 
@@ -103,7 +104,7 @@ jobs:
         with:
           fail_ci_if_error: false
 
-  resp3_tests:
+  resp3-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -144,7 +145,8 @@ jobs:
           name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}}-resp3
           path: |
             ${{matrix.test-type}}*-results.xml
-            ${{matrix.test-type}}-profile.out
+            prof/**
+            profile_output*
           if-no-files-found: error
           retention-days: 10
 

.gitignore

@@ -16,4 +16,6 @@ coverage.xml
 .venv*
 *.xml
 .coverage*
+prof
+profile_output*
 docker/stunnel/keys

dev_requirements.txt

@@ -10,6 +10,8 @@ packaging>=20.4
 pytest
 pytest-asyncio
 pytest-cov
+pytest-line-profiler
+pytest-profiling
 pytest-timeout
 ujson>=4.2.0
 urllib3<2

redis/cluster.py

@@ -1522,6 +1522,8 @@ def _get_or_create_cluster_node(self, host, port, role, tmp_nodes_cache):
                 target_node = ClusterNode(host, port, role)
             if target_node.server_type != role:
                 target_node.server_type = role
+            # add this node to the nodes cache
+            tmp_nodes_cache[target_node.name] = target_node
 
         return target_node
 
@@ -1585,31 +1587,26 @@ def initialize(self):
                 port = int(primary_node[1])
                 host, port = self.remap_host_port(host, port)
 
+                nodes_for_slot = []
+
                 target_node = self._get_or_create_cluster_node(
                     host, port, PRIMARY, tmp_nodes_cache
                 )
-                # add this node to the nodes cache
-                tmp_nodes_cache[target_node.name] = target_node
+                nodes_for_slot.append(target_node)
+
+                replica_nodes = [slot[j] for j in range(3, len(slot))]
+                for replica_node in replica_nodes:
+                    host = str_if_bytes(replica_node[0])
+                    port = int(replica_node[1])
+                    host, port = self.remap_host_port(host, port)
+                    target_replica_node = self._get_or_create_cluster_node(
+                        host, port, REPLICA, tmp_nodes_cache
+                    )
+                    nodes_for_slot.append(target_replica_node)
 
                 for i in range(int(slot[0]), int(slot[1]) + 1):
                     if i not in tmp_slots:
-                        tmp_slots[i] = []
-                        tmp_slots[i].append(target_node)
-                        replica_nodes = [slot[j] for j in range(3, len(slot))]
-
-                        for replica_node in replica_nodes:
-                            host = str_if_bytes(replica_node[0])
-                            port = replica_node[1]
-                            host, port = self.remap_host_port(host, port)
-
-                            target_replica_node = self._get_or_create_cluster_node(
-                                host, port, REPLICA, tmp_nodes_cache
-                            )
-                            tmp_slots[i].append(target_replica_node)
-                            # add this node to the nodes cache
-                            tmp_nodes_cache[target_replica_node.name] = (
-                                target_replica_node
-                            )
+                        tmp_slots[i] = nodes_for_slot
                     else:
                         # Validate that 2 nodes want to use the same slot cache
                         # setup

tasks.py

@@ -56,11 +56,11 @@ def standalone_tests(c, uvloop=False, protocol=2):
     """Run tests against a standalone redis instance"""
     if uvloop:
         run(
-            f"python -m cProfile -o standalone-profile.out -m pytest --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop --junit-xml=standalone-uvloop-results.xml"
+            f"pytest --profile --profile-svg --line-profile --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop --junit-xml=standalone-uvloop-results.xml"
         )
     else:
         run(
-            f"python -m cProfile -o standalone-profile.out -m pytest --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --junit-xml=standalone-results.xml"
+            f"pytest --profile --profile-svg --line-profile --protocol={protocol} --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --junit-xml=standalone-results.xml"
         )
 
 
@@ -70,11 +70,11 @@ def cluster_tests(c, uvloop=False, protocol=2):
     cluster_url = "redis://localhost:16379/0"
     if uvloop:
         run(
-            f"python -m cProfile -o cluster-profile.out -m pytest --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-uvloop-results.xml --uvloop"
+            f"pytest --profile --profile-svg --line-profile --protocol={protocol} --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-uvloop-results.xml --uvloop"
         )
     else:
         run(
-            f"python -m cProfile -o cluster-profile.out -m pytest --protocol={protocol} --cov=./ --cov-report=xml:coverage_clusteclient.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-results.xml"
+            f"pytest --profile --profile-svg --line-profile --protocol={protocol} --cov=./ --cov-report=xml:coverage_clusteclient.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={cluster_url} --junit-xml=cluster-results.xml"
         )