ci: parallel system tests (#8813)

gnufede · emmettbutler · web-flow · commit ad2fe2554a53 · 2024-04-01T10:13:21.000-07:00
CI: Introduces concurrency to System Tests execution in order to decrease wall time. Decreases wall time for system tests from ~37 min to ~23 min. ## Checklist - [x] Change(s) are motivated and described in the PR description - [x] Testing strategy is described if automated tests are not included in the PR - [x] Risks are described (performance impact, potential for breakage, maintainability) - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] [Library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) are followed or label `changelog/no-changelog` is set - [x] Documentation is included (in-code, generated user docs, [public corp docs](https://github.com/DataDog/documentation/)) - [x] Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) - [x] If this PR changes the public interface, I've notified `@DataDog/apm-tees`. - [x] If change touches code that signs or publishes builds or packages, or handles credentials of any kind, I've requested a review from `@DataDog/security-design-and-guidance`. ## Reviewer Checklist - [x] Title is accurate - [x] All changes are related to the pull request's stated goal - [x] Description motivates each change - [x] Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - [x] Testing strategy adequately addresses listed risks - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] Release note makes sense to a user of the library - [x] Author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - [x] Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Emmett Butler <723615+emmettbutler@users.noreply.github.com>
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -24,7 +24,7 @@ jobs:
           python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/system-tests.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))"
         continue-on-error: true
 
-  system-tests:
+  system-tests-build:
     runs-on: ubuntu-latest
     needs: needs-run
     strategy:
@@ -34,7 +34,7 @@ jobs:
           - weblog-variant: uwsgi-poc
           - weblog-variant: django-poc
           - weblog-variant: fastapi
-        #   runs django-poc for 3.12
+          # runs django-poc for 3.12
           - weblog-variant: python3.12
       fail-fast: false
     env:
@@ -47,6 +47,7 @@ jobs:
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     steps:
       - name: Setup python 3.9
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
@@ -69,100 +70,165 @@ jobs:
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         run: ./build.sh
 
-      - name: Run INTEGRATIONS
+      - name: Save
+        id: save
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh INTEGRATIONS
+        run: |
+          docker image save system_tests/weblog:latest | gzip > ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+          docker image save system_tests/agent:latest | gzip > ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
 
-      - name: Run CROSSED_TRACING_LIBRARIES
+      - uses: actions/upload-artifact@master
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh CROSSED_TRACING_LIBRARIES
+        with:
+          name: ${{ matrix.weblog-variant }}_${{ github.sha }}
+          path: |
+            ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+            ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
+            venv
+          retention-days: 2
 
-      - name: Run
+  system-tests:
+    runs-on: ubuntu-latest
+    needs: [needs-run, system-tests-build]
+    strategy:
+      matrix:
+        weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12]
+        scenario: [remote-config, appsec, appsec-1, other]
+
+      fail-fast: false
+    env:
+      TEST_LIBRARY: python
+      WEBLOG_VARIANT: ${{ matrix.weblog-variant }}
+      # system-tests requires an API_KEY, but it does not have to be a valid key, as long as we don't run a scenario
+      # that make assertion on backend data. Using a fake key allow to run system tests on PR originating from forks.
+      # If ever it's needed, a valid key exists in the repo, using ${{ secrets.DD_API_KEY }}
+      DD_API_KEY: 1234567890abcdef1234567890abcdef
+      CMAKE_BUILD_PARALLEL_LEVEL: 12
+    steps:
+      - name: Setup python 3.9
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
 
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+      - name: Checkout system tests
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+
+      - uses: actions/download-artifact@master
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        with:
+          name: ${{ matrix.weblog-variant }}_${{ github.sha }}
+          path: ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz
+
+      - name: docker load
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: |
+          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
+
+      - name: move venv
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: |
+          mv ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/venv venv
+          chmod -R +x venv/bin/*
+
+      - name: Run DEFAULT
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh DEFAULT
+
+      - name: Run SAMPLING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh SAMPLING
+
+      - name: Run INTEGRATIONS
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh INTEGRATIONS
+
+      - name: Run CROSSED_TRACING_LIBRARIES
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh CROSSED_TRACING_LIBRARIES
+
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
 
       - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
 
       - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
 
       - name: Run APPSEC_MISSING_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_MISSING_RULES
 
       - name: Run APPSEC_CUSTOM_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CUSTOM_RULES
 
       - name: Run APPSEC_CORRUPTED_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CORRUPTED_RULES
 
       - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS
 
-      - name: Run APPSEC_BLOCKING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_BLOCKING
-
-      - name: Run APPSEC_DISABLED
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_DISABLED
-
       - name: Run APPSEC_LOW_WAF_TIMEOUT
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_LOW_WAF_TIMEOUT
 
       - name: Run APPSEC_CUSTOM_OBFUSCATION
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CUSTOM_OBFUSCATION
 
       - name: Run APPSEC_RATE_LIMITER
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_RATE_LIMITER
 
-      - name: Run APPSEC_BLOCKING_FULL_DENYLIST
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
-
-      - name: Run APPSEC_REQUEST_BLOCKING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_REQUEST_BLOCKING
-
       - name: Run APPSEC_RUNTIME_ACTIVATION
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_RUNTIME_ACTIVATION
 
       - name: Run APPSEC_WAF_TELEMETRY
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_WAF_TELEMETRY
 
-      - name: Run SAMPLING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh SAMPLING
+      - name: Run APPSEC_DISABLED
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_DISABLED
+
+      - name: Run APPSEC_BLOCKING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+
+        run: ./run.sh APPSEC_BLOCKING
+      - name: Run APPSEC_BLOCKING_FULL_DENYLIST
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
+
+      - name: Run APPSEC_REQUEST_BLOCKING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_REQUEST_BLOCKING
 
-      # even on failures, we want to have artifact to be able to investigate, so run if build was a success
       # The compress step speed up a lot the upload artifact process
       - name: Compress artifact
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         id: compress-artifact
-        if: steps.build.outcome == 'success' || github.event_name == 'schedule'
         run: tar -czvf artifact.tar.gz $(ls | grep logs)
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         if: steps.compress-artifact.outcome == 'success' || github.event_name == 'schedule'
         with:
-          name: logs_${{ matrix.weblog-variant }}
+          name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }}
           path: artifact.tar.gz
 
+
   parametric:
     runs-on: ubuntu-latest
     needs: needs-run
@@ -203,3 +269,4 @@ jobs:
         with:
           name: logs_parametric
           path: artifact.tar.gz
+
