Merge pull request #4 from cybertec-postgresql/fix-workflow

Refresh testing harness at multiple places, remove some stubbornly broken parts for the meantime

Author: avandras

.github/workflows/install_deps.py

@@ -9,30 +9,32 @@
 
 
 def install_requirements(what):
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel', 'setuptools'])
+    if s != 0:
+        return s
+
     old_path = sys.path[:]
     w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
     sys.path.insert(0, os.path.dirname(os.path.dirname(w)))
     try:
         from setup import EXTRAS_REQUIRE, read
     finally:
         sys.path = old_path
-    requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
+    requirements = ['flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
     requirements += ['coverage']
     # try to split tests between psycopg2 and psycopg3
     requirements += ['psycopg[binary]'] if sys.version_info >= (3, 8, 0) and\
-        (sys.platform != 'darwin' or what == 'etcd3') else ['psycopg2-binary']
+        (sys.platform != 'darwin' or what == 'etcd3') else ['psycopg2-binary==2.9.9' 
+                                                            if sys.platform == 'darwin' else 'psycopg2-binary']
     for r in read('requirements.txt').split('\n'):
         r = r.strip()
         if r != '':
             extras = {e for e, v in EXTRAS_REQUIRE.items() if v and any(r.startswith(x) for x in v)}
             if not extras or what == 'all' or what in extras:
                 requirements.append(r)
 
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel'])
-    r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
-    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
-    return s | r
+    return subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
 
 
 def install_packages(what):
@@ -45,7 +47,7 @@ def install_packages(what):
     packages['exhibitor'] = packages['zookeeper']
     packages = packages.get(what, [])
     ver = versions.get(what)
-    if float(ver) >= 15:
+    if 15 <= float(ver) < 17:
         packages += ['postgresql-{0}-citus-12.1'.format(ver)]
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
     return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
@@ -142,4 +144,4 @@ def main():
 
 
 if __name__ == '__main__':
-    sys.exit(main())
+    sys.exit(main())

.github/workflows/run_tests.py

@@ -2,20 +2,20 @@
 import shutil
 import subprocess
 import sys
-import tempfile
 
 
 def main():
     what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all')
+    tmp = os.environ.get('RUNNER_TEMP')
 
     if what == 'all':
         flake8 = subprocess.call([sys.executable, 'setup.py', 'flake8'])
         test = subprocess.call([sys.executable, 'setup.py', 'test'])
         version = '.'.join(map(str, sys.version_info[:2]))
-        shutil.move('.coverage', os.path.join(tempfile.gettempdir(), '.coverage.' + version))
+        print(shutil.move('.coverage', '.coverage.' + version))
+        print(os.getcwd())
         return flake8 | test
     elif what == 'combine':
-        tmp = tempfile.gettempdir()
         for name in os.listdir(tmp):
             if name.startswith('.coverage.'):
                 shutil.move(os.path.join(tmp, name), name)

.github/workflows/tests.yaml

@@ -18,8 +18,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.7, 3.8, 3.9, '3.10', 3.11, 3.12]
+        python-version: [3.7, 3.8, 3.9, '3.10', 3.11]
         exclude:
+          - os: macos-latest
+            python-version: 3.7
           - os: ubuntu-latest
             python-version: 3.7
         include:
@@ -37,10 +39,37 @@ jobs:
       run: python .github/workflows/install_deps.py
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
+    - name: Upload logs from unit tests
+      uses: actions/upload-artifact@v4
+      with:
+        name: logs-${{ matrix.os }}-${{ matrix.python-version }}
+        path: ${{ github.workspace }}/.coverage.*
+        include-hidden-files: true
+        retention-days: 1
 
-  post-unit:
+  post-unit-coverage:
+    runs-on: ${{ matrix.os }}
     needs: unit
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+
+    - name: Download logs from unit tests
+      uses: actions/download-artifact@v4
+      with:
+        pattern: logs-${{ matrix.os }}-*
+        merge-multiple: true
     - name: Combine coverage
       run: python .github/workflows/run_tests.py combine
 
@@ -63,24 +92,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: [3.7, 3.12]
+        os: [ubuntu-22.04]
+        python-version: [3.13]
+        # python-version: [3.7, 3.13]
         dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
-        exclude:
-        - os: ubuntu-latest
-          python-version: 3.7
-        include:
-        - os: ubuntu-22.04
-          python-version: 3.7
-        - os: macos-latest
-          python-version: 3.8
-          dcs: raft
-        - os: macos-latest
-          python-version: 3.9
-          dcs: etcd
-        - os: macos-latest
-          python-version: 3.11
-          dcs: etcd3
+#        include:
+#        - os: macos-latest
+#          python-version: 3.8
+#          dcs: raft
+#        - os: macos-latest
+#          python-version: 3.9
+#          dcs: etcd
+#        - os: macos-latest
+#          python-version: 3.11
+#          dcs: etcd3
 
     steps:
     - uses: actions/checkout@v4
@@ -98,13 +123,13 @@ jobs:
         sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg'
         sudo sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list'
         sudo sh -c 'wget -qO - https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg'
-      if: matrix.os == 'ubuntu'
+      if: startsWith(matrix.os, 'ubuntu')
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
     - name: Run behave tests
       run: python .github/workflows/run_tests.py
     - name: Upload logs if behave failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs
@@ -121,7 +146,7 @@ jobs:
 
   coveralls-finish:
     name: Finalize coveralls.io
-    needs: unit
+    needs: post-unit-coverage
     runs-on: ubuntu-latest
     steps:
     - uses: actions/setup-python@v4
@@ -144,14 +169,14 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
 
     - name: Install dependencies
       run: python -m pip install -r requirements.txt psycopg2-binary psycopg
 
-    - uses: jakebailey/pyright-action@v1
+    - uses: jakebailey/pyright-action@v2
       with:
         version: 1.1.347
 

docs/conf.py

@@ -114,9 +114,6 @@
 
 html_theme = 'sphinx_rtd_theme'
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

patroni/ctl.py

@@ -41,8 +41,12 @@
     from psycopg import Cursor
     from psycopg2 import cursor
 
-try:
-    from ydiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
+try:  # pragma: no cover
+    from ydiff import markup_to_pager  # pyright: ignore [reportMissingModuleSource]
+    try:
+        from ydiff import PatchStream  # pyright: ignore [reportMissingModuleSource]
+    except ImportError:
+        PatchStream = iter
 except ImportError:  # pragma: no cover
     from cdiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
 

patroni/log.py

@@ -351,7 +351,7 @@ def _get_json_formatter(self, logformat: type_logformat, dateformat: Optional[st
         try:
             from pythonjsonlogger import jsonlogger
 
-            return jsonlogger.JsonFormatter(
+            return jsonlogger.JsonFormatter(  # pyright: ignore [reportPrivateImportUsage]
                 jsonformat,
                 dateformat,
                 rename_fields=rename_fields,

requirements.docs.txt

@@ -1,4 +1,4 @@
-sphinx>=4
+sphinx>=4,<8.2.0
 sphinx_rtd_theme>1
 sphinxcontrib-apidoc
 sphinx-github-style<1.0.3

requirements.txt

@@ -10,5 +10,6 @@ python-dateutil
 pysyncobj>=0.3.8
 cryptography>=1.4
 psutil>=2.0.0
-ydiff>=1.2.0
+ydiff>=1.2.0,<1.5,!=1.4.0,!=1.4.1
 python-json-logger>=2.0.2
+setuptools

tests/__init__.py

@@ -3,7 +3,7 @@
 import shutil
 import unittest
 
-from mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, PropertyMock, patch
 
 import urllib3
 

tests/test_api.py

@@ -5,7 +5,7 @@
 
 from http.server import HTTPServer
 from io import BytesIO as IO
-from mock import Mock, PropertyMock, patch
+from unittest.mock import Mock, PropertyMock, patch
 from socketserver import ThreadingMixIn
 
 from patroni import global_config