chore: add fuzzy tests (hypothesis) of output_format (#74)

Co-authored-by: Michał Sośnicki <michal.sosnicki@neptune.ai>

Author: michalsosn

.github/actions/run-tests/action.yml

@@ -15,6 +15,10 @@ inputs:
     description: 'Working directory'
     required: false
     default: .
+  timeout:
+    description: "Test timeout in seconds"
+    required: false
+    default: "120"
 
 runs:
   using: "composite"
@@ -23,7 +27,7 @@ runs:
       working-directory: ${{ inputs.working-directory }}
       run: |
         pytest -v "./tests/${{ inputs.test-directory }}" \
-        --timeout=120 --timeout_method=thread \
+        --timeout=${{ inputs.timeout }} --timeout_method=thread \
         --color=yes \
         --junitxml="${{ inputs.working-directory }}/test-results/test-${{ inputs.report-suffix }}.xml"
       shell: bash

.github/workflows/tests-e2e.yml

@@ -17,7 +17,7 @@ on:
   pull_request: # Don't run e2e tests on PRs that don't modify source code
     paths:
       - 'src/**'
-      - 'tests/**'
+      - 'tests/e2e/**'
       - 'dev_requirements.txt'
       - 'pyproject.toml'
       - '.github/workflows/tests-e2e.yml'

.github/workflows/tests-fuzzy.yml

@@ -0,0 +1,71 @@
+name: Run fuzzy tests - neptune-query
+
+on:
+  schedule:
+    - cron: '0 8 * * *' # Run at 8:00 daily
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - dev/.*
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'tests/fuzzy/**'
+      - 'dev_requirements.txt'
+      - 'pyproject.toml'
+      - '.github/workflows/tests-fuzzy.yml'
+
+jobs:
+  test:
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.13" ]
+        os: [ ubuntu ]
+
+    name: 'fuzzy test (${{ matrix.os }} - py${{ matrix.python-version }})'
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+
+      - name: Install package
+        uses: ./.github/actions/install-package
+        with:
+          python-version: ${{ matrix.python-version }}
+          os: ${{ matrix.os }}-latest
+
+      - name: Run tests
+        uses: ./.github/actions/run-tests
+        with:
+          test-directory: fuzzy
+          report-job: 'test fuzzy (${{ matrix.os }} - py${{ matrix.python-version }})'
+          report-suffix: 'fuzzy-${{ matrix.python-version }}-${{ matrix.os }}'
+          timeout: "900"
+        env:
+          NEPTUNE_E2E_HYPOTHESIS_PROFILE: ${{ github.event_name == 'schedule' && 'ci-nightly' || 'ci-quick' }}
+
+      - name: Notify Slack on failure
+        if: failure() && github.event_name == 'schedule'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const webhookUrl = process.env.SLACK_WEBHOOK_URL;
+            const payload = {
+              username: "GitHub Actions",
+              text: `Fuzzy tests failed in the repository: <https://github.com/${context.repo.owner}/${context.repo.repo}|${context.repo.owner}/${context.repo.repo}>. Please check the details.`
+            };
+            await fetch(webhookUrl, {
+              method: 'POST',
+              headers: {
+                'Content-Type': 'application/json'
+              },
+              body: JSON.stringify(payload)
+            });
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/tests-unit.yml

@@ -6,9 +6,10 @@ on:
   pull_request:
     paths:
       - 'src/**'
-      - 'tests/**'
+      - 'tests/unit/**'
       - 'dev_requirements.txt'
       - 'pyproject.toml'
+      - '.github/workflows/tests-unit.yml'
   push:
     branches:
       - main

.gitignore

@@ -123,6 +123,9 @@ venv.bak/
 .history
 .vscode
 
+# Cursor
+.cursor/
+
 stream.bin
 
 # mocks

dev_requirements.txt

@@ -10,3 +10,4 @@ pytest-timeout
 pytest-xdist
 icecream
 git+https://github.com/neptune-ai/neptune-client-scale.git@main#egg=neptune-scale
+hypothesis==6.139.2

tests/fuzzy/__init__.py

@@ -0,0 +1,31 @@
+import os
+from datetime import timedelta
+
+from hypothesis import settings
+
+
+def pytest_set_filtered_exceptions() -> list[type[BaseException]]:
+    class DoNotFilterAnythingMarker(Exception):
+        pass
+
+    return [DoNotFilterAnythingMarker]
+
+
+settings.register_profile(
+    "ci-quick",
+    settings.get_profile("ci"),
+    max_examples=100,
+    deadline=timedelta(seconds=10),
+    derandomize=True,
+)
+
+
+settings.register_profile(
+    "ci-nightly",
+    settings.get_profile("ci"),
+    max_examples=2000,
+    deadline=timedelta(seconds=20),
+    derandomize=False,
+)
+
+settings.load_profile(os.getenv("NEPTUNE_E2E_HYPOTHESIS_PROFILE", "ci-quick"))