Support for protocol version (RESP3) (#190)

Support for protocol version (RESP3)
* Added `protocol=n` argument to environment classes
* Added `Defaults.env_factory` to allow default environment customization (e.g. in RedisTimeSeries)
* Extended `terminateRetries` support to `Defaults` and `RedisCluster`
* Aviod "UnicodeEncodeError: 'ascii' codec can't encode character" errors by reopening stdout/err with utf8 encoding
* CI: solve redis 7.2 build problem on macOS by using `getredis` from Readies

Author: rafie

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
   - cron: "0 0 * * *"
 
 jobs:
-  build-ubuntu:
+  build:
     name: Test on ${{ matrix.platform }} with Python ${{ matrix.python }}
     runs-on: ${{ matrix.platform }}
     timeout-minutes: 40
@@ -18,6 +18,9 @@ jobs:
         platform: ['ubuntu-20.04', 'macos-11']
         python: ['3.7', '3.8', '3.9', '3.10', '3.11']
       fail-fast: false
+    defaults:
+      run:
+        shell: bash -l -eo pipefail {0}
 
     steps:
     - uses: actions/checkout@v3
@@ -40,20 +43,23 @@ jobs:
         restore-keys: |
           ${{ matrix.platform }}-${{ matrix.python }}-pyproject.toml-${{hashFiles('pyproject.toml')}}}
 
+    - name: Setup automation
+      run: |
+        git clone --recursive https://github.com/RedisLabsModules/readies.git
+        ./readies/bin/getpy3
+
     - name: Install Python dependencies
       run: |
-        pip install poetry
-        poetry config virtualenvs.create false
-        poetry export --dev --without-hashes -o requirements-${{matrix.platform}}-${{matrix.python}}.txt
-        pip install -r requirements-${{matrix.platform}}-${{matrix.python}}.txt
+        python3 -m pip install poetry
+        python3 -m poetry config virtualenvs.create false
+        python3 -m poetry export --dev --without-hashes -o requirements-${{matrix.platform}}-${{matrix.python}}.txt
+        python3 -m pip install -r requirements-${{matrix.platform}}-${{matrix.python}}.txt
 
     - name: Install Redis Server test dependencies
-      # if: steps.cache-redis.outputs.cache-hit != 'true'
       run: |
-        git clone https://github.com/redis/redis.git --branch 6.2.7 --depth 1
-        cd redis
-        make BUILD_TLS=yes -j
-        ./src/redis-server --version
+        ./readies/bin/getredis --workdir /tmp/redis -v 7 --keep
+        ln -s /tmp/redis/redis redis
+        redis-server --version
 
     - name: Generate test certificates
       run: |
@@ -64,109 +70,115 @@ jobs:
     - name: Unit Test with pytest
       timeout-minutes: 30
       run: |
-        TLS_CERT=./redis/tests/tls/redis.crt \
-        TLS_KEY=./redis/tests/tls/redis.key \
-        TLS_CACERT=./redis/tests/tls/ca.crt \
-        REDIS_BINARY=./redis/src/redis-server \
+        TLS="$PWD/redis/tests/tls"
+        TLS_CERT=$TLS/redis.crt \
+        TLS_KEY=$TLS/redis.key \
+        TLS_CACERT=$TLS/ca.crt \
+        REDIS_BINARY=`command -v redis-server` \
         pytest --ignore=tests/flow --ignore=test_example.py -v
 
     - name: Install RLTest
       run: |
-        pip install .
+        python3 -m pip install .
 
     - name: Flow Test OSS Single Module
       run: |
         cd tests/flow
         make -C modules
-        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server --module modules/module1.so
+        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs --module modules/module1.so
 
     - name: Flow Test OSS Multiple Modules  --use-slaves
       run: |
         cd tests/flow
         make -C modules
-        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server \
-          --module modules/module1.so --module-args '' --module modules/module2.so --module-args '' \
+        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs \
+          --module modules/module1.so --module-args '' \
+          --module modules/module2.so --module-args '' \
           --use-slaves
 
     - name: Flow Test OSS Multiple Modules  --use-aof
       run: |
         cd tests/flow
         make -C modules
-        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server \
-          --module modules/module1.so --module-args '' --module modules/module2.so --module-args '' \
+        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs \
+          --module modules/module1.so --module-args '' \
+          --module modules/module2.so --module-args '' \
           --use-aof
 
     - name: Flow Test OSS Multiple Modules
       run: |
         cd tests/flow
         make -C modules
-        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server \
-          --module modules/module1.so --module-args '' --module modules/module2.so --module-args ''
+        CONTAINS_MODULES=1 RLTest --env oss -v --clear-logs \
+          --module modules/module1.so --module-args '' \
+          --module modules/module2.so --module-args ''
 
     - name: Flow Test OSS-CLUSTER Modules
       run: |
         cd tests/flow
         make -C modules
-        CONTAINS_MODULES=1 RLTest --env oss-cluster -v --clear-logs --oss-redis-path ../../redis/src/redis-server \
-          --module modules/module1.so --module-args '' --module modules/module2.so --module-args ''
+        CONTAINS_MODULES=1 RLTest --env oss-cluster -v --clear-logs \
+          --module modules/module1.so --module-args '' \
+          --module modules/module2.so --module-args ''
 
     - name: Flow Test OSS TCP
       run: |
         cd tests/flow
-        RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server
+        RLTest --env oss -v --clear-logs
 
     - name: Flow Test OSS UNIX SOCKETS
       run: |
         cd tests/flow
-        RLTest --env oss -v --clear-logs --oss-redis-path ../../redis/src/redis-server
+        RLTest --env oss -v --clear-logs
 
     - name: Flow Test OSS TCP SLAVES
       run: |
         cd tests/flow
-        RLTest --env oss -v --unix --clear-logs --oss-redis-path ../../redis/src/redis-server
+        RLTest --env oss -v --unix --clear-logs
 
     - name: Flow Test OSS-CLUSTER TCP
       run: |
         cd tests/flow
-        RLTest --env oss-cluster -v --clear-logs --shards-count 3 --oss-redis-path ../../redis/src/redis-server
+        RLTest --env oss-cluster -v --clear-logs --shards-count 3
 
     - name: Flow Test OSS TCP with TLS
       run: |
+        TLS="$PWD/redis/tests/tls"
         cd tests/flow
         RLTest --env oss -v --clear-logs \
-        --oss-redis-path ../../redis/src/redis-server \
-        --tls-cert-file ../../redis/tests/tls/redis.crt \
-        --tls-key-file ../../redis/tests/tls/redis.key \
-        --tls-ca-cert-file ../../redis/tests/tls/ca.crt \
+        --tls-cert-file $TLS/redis.crt \
+        --tls-key-file $TLS/redis.key \
+        --tls-ca-cert-file $TLS/ca.crt \
         --tls
 
     - name: Flow Test OSS-CLUSTER with TLS
       run: |
+        TLS="$PWD/redis/tests/tls"
         cd tests/flow
         RLTest --env oss-cluster --shards-count 3 -v --clear-logs \
-        --oss-redis-path ../../redis/src/redis-server \
-        --tls-cert-file ../../redis/tests/tls/redis.crt \
-        --tls-key-file ../../redis/tests/tls/redis.key \
-        --tls-ca-cert-file ../../redis/tests/tls/ca.crt \
+        --tls-cert-file $TLS/redis.crt \
+        --tls-key-file $TLS/redis.key \
+        --tls-ca-cert-file $TLS/ca.crt \
         --tls
 
     - name: Flow Test OSS-CLUSTER with SLAVES and TLS
       run: |
+        TLS="$PWD/redis/tests/tls"
         cd tests/flow
         RLTest --env oss-cluster --shards-count 3 --use-slaves -v --clear-logs \
-        --oss-redis-path ../../redis/src/redis-server \
-        --tls-cert-file ../../redis/tests/tls/redis.crt \
-        --tls-key-file ../../redis/tests/tls/redis.key \
-        --tls-ca-cert-file ../../redis/tests/tls/ca.crt \
+        --tls-cert-file $TLS/redis.crt \
+        --tls-key-file $TLS/redis.key \
+        --tls-ca-cert-file $TLS/ca.crt \
         --tls
 
     - name: Generate coverage report
       if: matrix.python == '3.9' && matrix.platform != 'macos-11'
       run: |
-        TLS_CERT=./redis/tests/tls/redis.crt \
-        TLS_KEY=./redis/tests/tls/redis.key \
-        TLS_CACERT=./redis/tests/tls/ca.crt \
-        REDIS_BINARY=./redis/src/redis-server \
+        TLS="$PWD/redis/tests/tls"
+        TLS_CERT=$TLS/redis.crt \
+        TLS_KEY=$TLS/redis.key \
+        TLS_CACERT=$TLS/ca.crt \
+        REDIS_BINARY=`command -v redis-server` \
         pytest --ignore=tests/flow --ignore=test_example.py --cov-config=.coveragerc --cov-report=xml --cov=RLTest
 
     - name: Upload coverage to Codecov

RLTest/__main__.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import argparse
+import io
 import os
 import cmd
 import traceback
@@ -587,10 +588,11 @@ def _runTest(self, test, numberOfAssertionFailed=0, prefix='', before=None, afte
             test_args = inspect.getargspec(test.target).args
         except:
             test_args = inspect.getfullargspec(test.target).args
-        
+
         if len(test_args) > 0 and not test.is_method:
             try:
-                env = Env(testName=test.name)
+                # env = Env(testName=test.name)
+                env = Defaults.env_factory(testName=test.name)
             except Exception as e:
                 self.handleFailure(testFullName=testFullName, exception=e, prefix=msgPrefix, testname=test.name)
                 return 0
@@ -674,7 +676,8 @@ def execute(self):
         Env.RTestInstance = self
         if self.args.env_only:
             Defaults.verbose = 2
-            env = Env(testName='manual test env')
+            # env = Env(testName='manual test env')
+            env = Defaults.env_factory(testName='manual test env')
             if self.args.interactive_debugger:
                 while env.isUp():
                     time.sleep(1)
@@ -786,6 +789,9 @@ def run_jobs(jobs, results, port):
 
 
 def main():
+    # Aviod "UnicodeEncodeError: 'ascii' codec can't encode character" errors
+    sys.stdout = io.open(sys.stdout.fileno(), 'w', encoding='utf8')
+    sys.stderr = io.open(sys.stderr.fileno(), 'w', encoding='utf8')
     RLTest().execute()
 
 

RLTest/env.py

@@ -112,6 +112,7 @@ class Defaults:
     module_args = None
 
     env = 'oss'
+    env_factory = lambda *args, **kwargs: Env(*args, **kwargs)
     binary = 'redis-server'
     proxy_binary = None
     re_binary = None
@@ -143,6 +144,8 @@ class Defaults:
     curr_test_name = None
     port=6379
     enable_debug_command=False
+    terminate_retries=None
+    terminate_retry_secs=None
 
     def getKwargs(self):
         kwargs = {
@@ -163,15 +166,17 @@ def getKwargs(self):
             'tlsKeyFile': self.tls_key_file,
             'tlsCaCertFile': self.tls_ca_cert_file,
             'tlsPassphrase': self.tls_passphrase,
-            'password': self.oss_password
+            'password': self.oss_password,
+            'terminateRetries': self.terminate_retries,
+            'terminateRetrySecs': self.terminate_retry_secs,
         }
         return kwargs
 
 
 class Env:
     RTestInstance = None
     EnvCompareParams = ['module', 'moduleArgs', 'env', 'useSlaves', 'shardsCount', 'useAof',
-                        'useRdbPreamble', 'forceTcp', 'enableDebugCommand']
+                        'useRdbPreamble', 'forceTcp', 'enableDebugCommand', 'protocol']
 
     def compareEnvs(self, env):
         if env is None:
@@ -186,7 +191,7 @@ def __init__(self, testName=None, testDescription=None, module=None,
                  useAof=None, useRdbPreamble=None, forceTcp=False, useTLS=False, tlsCertFile=None, tlsKeyFile=None,
                  tlsCaCertFile=None, tlsPassphrase=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
                  redisEnterpriseBinaryPath=None, noDefaultModuleArgs=False, clusterNodeTimeout = None,
-                 freshEnv=False, enableDebugCommand=None):
+                 freshEnv=False, enableDebugCommand=None, protocol=2, terminateRetries=None, terminateRetrySecs=None):
 
         self.testName = testName if testName else Defaults.curr_test_name
         if self.testName is None:
@@ -224,10 +229,14 @@ def __init__(self, testName=None, testDescription=None, module=None,
         self.clusterNodeTimeout = clusterNodeTimeout if clusterNodeTimeout else Defaults.cluster_node_timeout
         self.port = Defaults.port
         self.enableDebugCommand = enableDebugCommand if enableDebugCommand else Defaults.enable_debug_command
+        self.terminateRetries = terminateRetries
+        self.terminateRetrySecs = terminateRetrySecs
+
+        self.protocol = protocol
 
         self.assertionFailedSummary = []
 
-        if (not freshEnv) and Env.RTestInstance and Env.RTestInstance.currEnv and self.compareEnvs(Env.RTestInstance.currEnv):
+        if not freshEnv and Env.RTestInstance and Env.RTestInstance.currEnv and self.compareEnvs(Env.RTestInstance.currEnv):
             self.envRunner = Env.RTestInstance.currEnv.envRunner
         else:
             if Env.RTestInstance and Env.RTestInstance.currEnv:
@@ -324,7 +333,10 @@ def getEnvKwargs(self):
             'clusterNodeTimeout': self.clusterNodeTimeout,
             'tlsPassphrase': self.tlsPassphrase,
             'port': self.port,
-            'enableDebugCommand': self.enableDebugCommand
+            'enableDebugCommand': self.enableDebugCommand,
+            'protocol': self.protocol,
+            'terminateRetries': self.terminateRetries,
+            'terminateRetrySecs': self.terminateRetrySecs,
         }
         return kwargs
 

RLTest/redis_cluster.py

@@ -20,6 +20,9 @@ def __init__(self, **kwargs):
         self.useTLS = kwargs['useTLS']
         self.decodeResponses = kwargs.get('decodeResponses', False)
         self.tlsPassphrase = kwargs.get('tlsPassphrase', None)
+        self.protocol = kwargs.get('protocol', 2)
+        self.terminateRetries = kwargs.get('terminateRetries', None)
+        self.terminateRetrySecs = kwargs.get('terminateRetrySecs', None)
         startPort = kwargs.pop('port', 10000)
         totalRedises = self.shardsCount * (2 if useSlaves else 1)
         randomizePorts = kwargs.pop('randomizePorts', False)
@@ -42,7 +45,6 @@ def printEnvData(self, prefix=''):
             shard.printEnvData(prefix + '\t')
 
     def waitCluster(self, timeout_sec=40):
-
         st = time.time()
         ok = 0
 
@@ -129,12 +131,16 @@ def getClusterConnection(self):
                 ssl_password=self.tlsPassphrase,
                 password=self.password,
                 startup_nodes=statupNode,
-                decode_responses=self.decodeResponses
+                decode_responses=self.decodeResponses,
+                protocol=self.protocol,
+                terminateRetries=self.terminateRetries, terminateRetrySecs=self.terminateRetrySecs
             )
         else:
             return redis.RedisCluster(
                 startup_nodes=statupNode,
-                decode_responses=self.decodeResponses, password=self.password)
+                decode_responses=self.decodeResponses, password=self.password,
+                protocol=self.protocol,
+                terminateRetries=self.terminateRetries, terminateRetrySecs=self.terminateRetrySecs)
 
     def getSlaveConnection(self):
         raise Exception('unsupported')