From 58427867ecc0fc7032c6d03186da30bd88279e41 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 14:41:00 -0400 Subject: [PATCH 01/42] Try datadog test optimization --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f419fb9..dac151bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,3 +68,9 @@ jobs: run: | dist/datadog-sync diffs --validate=false --verify-ddr-status=false exit $? + - name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v2 + with: + languages: python + api_key: ${{secrets.DD_API_KEY}} + site: datadoghq.com From d7f57994b834988aa73c2b623937e56fd6c8eea9 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 14:48:22 -0400 Subject: [PATCH 02/42] move --- .../workflows/datadog_test_optimization.yml | 32 +++++++++++++++++++ .github/workflows/test.yml | 6 ---- 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/datadog_test_optimization.yml diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml new file mode 100644 index 00000000..6ff91a23 --- /dev/null +++ b/.github/workflows/datadog_test_optimization.yml @@ -0,0 +1,32 @@ +name: Datadog Test Optimization + +permissions: + checks: write + contents: read + +on: + pull_request: + branches: + - main + types: + - labeled + - unlabeled + - opened + - synchronize + - reopened + +jobs: + test-optimization: + runs-on: ubuntu-latest + services: + datadog-agent: + image: datadog/agent:latest + ports: + - 8126:8126 + steps: + - name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v2 + with: + languages: python + api_key: ${{secrets.DD_API_KEY}} + site: datadoghq.com diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dac151bb..6f419fb9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,9 +68,3 @@ jobs: run: | dist/datadog-sync diffs --validate=false --verify-ddr-status=false exit $? - - name: Configure Datadog Test Optimization - uses: datadog/test-visibility-github-action@v2 - with: - languages: python - api_key: ${{secrets.DD_API_KEY}} - site: datadoghq.com From 266ea37da86f040dcf502f6fbdc827a93583e961 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 14:52:33 -0400 Subject: [PATCH 03/42] on push --- .github/workflows/datadog_test_optimization.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 6ff91a23..82669dce 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -4,16 +4,9 @@ permissions: checks: write contents: read -on: - pull_request: - branches: - - main - types: - - labeled - - unlabeled - - opened - - synchronize - - reopened +env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} +on: push jobs: test-optimization: From 58c80e01fd12051806c142be6b72dbb64755aa11 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:01:46 -0400 Subject: [PATCH 04/42] format --- .github/workflows/datadog_test_optimization.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 82669dce..57d6b2d4 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -4,9 +4,10 @@ permissions: checks: write contents: read +on: push + env: DD_API_KEY: ${{ secrets.DD_API_KEY }} -on: push jobs: test-optimization: From 50af3a0d36edd1b46131582ff1fcb4280d071e0e Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:03:27 -0400 Subject: [PATCH 05/42] Swich to env --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 57d6b2d4..a7a7ba68 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -22,5 +22,5 @@ jobs: uses: datadog/test-visibility-github-action@v2 with: languages: python - api_key: ${{secrets.DD_API_KEY}} + api_key: ${{env.DD_API_KEY}} site: datadoghq.com From d7e75279468b4e9175a6d802104b981afcad85c3 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:04:55 -0400 Subject: [PATCH 06/42] Remove agent --- .github/workflows/datadog_test_optimization.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index a7a7ba68..835be8e7 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -12,11 +12,6 @@ env: jobs: test-optimization: runs-on: ubuntu-latest - services: - datadog-agent: - image: datadog/agent:latest - ports: - - 8126:8126 steps: - name: Configure Datadog Test Optimization uses: datadog/test-visibility-github-action@v2 From 8dfb90798c99d36eb84db90314c9ede39b82bcca Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:27:51 -0400 Subject: [PATCH 07/42] install requirements --- .github/workflows/datadog_test_optimization.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 835be8e7..80700902 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -13,6 +13,10 @@ jobs: test-optimization: runs-on: ubuntu-latest steps: + - name: Install requirements + run: | + python -m pip install . + python -m pip install -r scripts/test_requirements.txt - name: Configure Datadog Test Optimization uses: datadog/test-visibility-github-action@v2 with: From 2e320bcd242cfb27f2cf127c7b10a57ed4c8d805 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:29:53 -0400 Subject: [PATCH 08/42] Add checkout --- .github/workflows/datadog_test_optimization.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 80700902..0301eb26 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -13,6 +13,12 @@ jobs: test-optimization: runs-on: ubuntu-latest steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Checkout code + uses: actions/checkout@v2 - name: Install requirements run: | python -m pip install . From 0da283ce25f3b1a64147969a1c3f6a8812bf9dd4 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:31:14 -0400 Subject: [PATCH 09/42] Need this --- scripts/test_requirements.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 scripts/test_requirements.txt diff --git a/scripts/test_requirements.txt b/scripts/test_requirements.txt new file mode 100644 index 00000000..f4996d3d --- /dev/null +++ b/scripts/test_requirements.txt @@ -0,0 +1,18 @@ +boto3==1.35.91 +click==8.1.7 +configobj==5.0.8 +deepdiff==6.7.1 +aiohttp==3.9.5 +tqdm==4.66.2 +certifi>=2022.12.7 +python-dateutil +setuptools>=67.6.0 +setuptools_scm +ddtrace==2.21.1 +black==24.3.0 +pytest==8.1.1 +pytest-black +pytest-console-scripts +pytest-recording +vcrpy@git+https://github.com/skarimo/vcrpy.git +freezegun==1.5.0 From ceb9ec30020dfafe150c0e4a7d6a51df45944cad Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:33:35 -0400 Subject: [PATCH 10/42] actually run the tests --- .github/workflows/datadog_test_optimization.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 0301eb26..d4f41a26 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -29,3 +29,6 @@ jobs: languages: python api_key: ${{env.DD_API_KEY}} site: datadoghq.com + - name: Run tests + shell: bash + run: pytest From 28cfd17d69ec4e494a1ed0a6acde5101e6c3c544 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:36:41 -0400 Subject: [PATCH 11/42] Fix indent --- .github/workflows/datadog_test_optimization.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index d4f41a26..732d427b 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -29,6 +29,6 @@ jobs: languages: python api_key: ${{env.DD_API_KEY}} site: datadoghq.com - - name: Run tests - shell: bash - run: pytest + - name: Run tests + shell: bash + run: pytest From c67e078a363e3b46f5142faf04589902857e634b Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:40:04 -0400 Subject: [PATCH 12/42] record --- .github/workflows/datadog_test_optimization.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 732d427b..592e0911 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -8,6 +8,12 @@ on: push env: DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_DESTINATION_API_KEY: ${{ secrets.DD_DESTINATION_API_KEY }} + DD_DESTINATION_APP_KEY: ${{ secrets.DD_DESTINATION_APP_KEY }} + DD_DESTINATION_API_URL: ${{ secrets.DD_DESTINATION_API_URL }} + DD_SOURCE_API_KEY: ${{ secrets.DD_SOURCE_API_KEY }} + DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} + DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} jobs: test-optimization: @@ -31,4 +37,4 @@ jobs: site: datadoghq.com - name: Run tests shell: bash - run: pytest + run: RECORD=true pytest From 553a0ff90bf52fc51d7b4178b87a042d25c0b53e Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:45:28 -0400 Subject: [PATCH 13/42] Move record --- .github/workflows/datadog_test_optimization.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 592e0911..df95199e 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -14,6 +14,7 @@ env: DD_SOURCE_API_KEY: ${{ secrets.DD_SOURCE_API_KEY }} DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} + RECORD: true jobs: test-optimization: @@ -37,4 +38,4 @@ jobs: site: datadoghq.com - name: Run tests shell: bash - run: RECORD=true pytest + run: pytest From beecfa46fee643229d2a6b2aef4e722ab9c5cccf Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:53:44 -0400 Subject: [PATCH 14/42] Turn of record --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index df95199e..2eef97da 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -14,7 +14,7 @@ env: DD_SOURCE_API_KEY: ${{ secrets.DD_SOURCE_API_KEY }} DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} - RECORD: true + RECORD: none jobs: test-optimization: From 524e7fe20ee46d069abf70fb503732338c0d3379 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 15:59:19 -0400 Subject: [PATCH 15/42] Turn off recording --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 2eef97da..084da76e 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -14,7 +14,7 @@ env: DD_SOURCE_API_KEY: ${{ secrets.DD_SOURCE_API_KEY }} DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} - RECORD: none + RECORD: false jobs: test-optimization: From 918db69ea8be0f2492fb1d773a638516898faf35 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 16:35:57 -0400 Subject: [PATCH 16/42] switch to sha --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 084da76e..41d54f76 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Python - uses: actions/setup-python@v4 + uses: actions/setup-python@f03e505388af670b5a108629e0ba26befc08d5c62b41f46146a45fe29ae509a5 with: python-version: 3.12 - name: Checkout code From 823b3f7ac6dbe5cc2ed5d5500f02ce8412992c89 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 16:37:30 -0400 Subject: [PATCH 17/42] add sha256: --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 41d54f76..ca5ef3e1 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Python - uses: actions/setup-python@f03e505388af670b5a108629e0ba26befc08d5c62b41f46146a45fe29ae509a5 + uses: actions/setup-python@sha256:f03e505388af670b5a108629e0ba26befc08d5c62b41f46146a45fe29ae509a5 with: python-version: 3.12 - name: Checkout code From 76d77d26db276ad646633bfc17f76a30c51bdb8d Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 16:40:42 -0400 Subject: [PATCH 18/42] Version --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index ca5ef3e1..31a09335 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Python - uses: actions/setup-python@sha256:f03e505388af670b5a108629e0ba26befc08d5c62b41f46146a45fe29ae509a5 + uses: actions/setup-python@4.9.1 with: python-version: 3.12 - name: Checkout code From d6f69360b08bd77bfb8d4bd29eac01c12b85eefc Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 16:48:38 -0400 Subject: [PATCH 19/42] Add version --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 31a09335..0739165d 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -31,7 +31,7 @@ jobs: python -m pip install . python -m pip install -r scripts/test_requirements.txt - name: Configure Datadog Test Optimization - uses: datadog/test-visibility-github-action@v2 + uses: datadog/test-visibility-github-action@v2.3.4 with: languages: python api_key: ${{env.DD_API_KEY}} From f558a13cf8c81c79c9d570377dce34d8685c53d9 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 17:00:03 -0400 Subject: [PATCH 20/42] Try sha again --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 0739165d..1d2e3a08 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Python - uses: actions/setup-python@4.9.1 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c with: python-version: 3.12 - name: Checkout code From 64380b22ca2ea0882aa2f952fc654736bb4507f5 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 17:01:55 -0400 Subject: [PATCH 21/42] sha for this --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 1d2e3a08..87bd6844 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -31,7 +31,7 @@ jobs: python -m pip install . python -m pip install -r scripts/test_requirements.txt - name: Configure Datadog Test Optimization - uses: datadog/test-visibility-github-action@v2.3.4 + uses: datadog/test-visibility-github-action@fa5d4c02a5f2329cf2c62a70fd8ec532317af1c9 with: languages: python api_key: ${{env.DD_API_KEY}} From 7ddbee72a89d72c11499d505f38c89144c61b5a5 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 17 Apr 2025 17:04:48 -0400 Subject: [PATCH 22/42] sha for checkout --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 87bd6844..1a2f0928 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -25,7 +25,7 @@ jobs: with: python-version: 3.12 - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Install requirements run: | python -m pip install . From c835de1b706e5dc7fb7cdf542b20360d50e40e2a Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Fri, 18 Apr 2025 09:58:57 -0400 Subject: [PATCH 23/42] Test config From defdb83de6173f8b577b127312ef35e6aa3b7de2 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 12:55:36 -0400 Subject: [PATCH 24/42] Change this to scheduled --- .github/workflows/datadog_test_optimization.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 1a2f0928..1deca64f 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -4,7 +4,10 @@ permissions: checks: write contents: read -on: push +on: + schedule: + # every hour, 9-5 eastern, M-F + - cron: '0 12-21 * * 1-5' env: DD_API_KEY: ${{ secrets.DD_API_KEY }} From 4308c0b848ddfb32c2af7de2ecac87afbbe00b35 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 12:59:28 -0400 Subject: [PATCH 25/42] They specifically recommend not running at the 0 minute --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 1deca64f..2bba7f5f 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -7,7 +7,7 @@ permissions: on: schedule: # every hour, 9-5 eastern, M-F - - cron: '0 12-21 * * 1-5' + - cron: '7 12-21 * * 1-5' env: DD_API_KEY: ${{ secrets.DD_API_KEY }} From 516278c95e79345a6b20319605c2534d14854ed9 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 13:05:43 -0400 Subject: [PATCH 26/42] Run on mac and windows too --- .github/workflows/datadog_test_optimization.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 2bba7f5f..a7a2a601 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -21,7 +21,10 @@ env: jobs: test-optimization: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: Install Python uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c From b8a93aab3442a7347c9912549365550926d1c76f Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 13:09:03 -0400 Subject: [PATCH 27/42] Trigger on push --- .github/workflows/datadog_test_optimization.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index a7a2a601..a57c8685 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -9,6 +9,11 @@ on: # every hour, 9-5 eastern, M-F - cron: '7 12-21 * * 1-5' + push: + branches: + - 'main' + - 'michael.richey/dd-test-opt' + env: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_DESTINATION_API_KEY: ${{ secrets.DD_DESTINATION_API_KEY }} From b846f5d7938406aa47090974a19e8eaae2a1afac Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 13:10:46 -0400 Subject: [PATCH 28/42] Refine the schedule and remove my branch from the push trigger --- .github/workflows/datadog_test_optimization.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index a57c8685..33e28d0f 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -7,12 +7,11 @@ permissions: on: schedule: # every hour, 9-5 eastern, M-F - - cron: '7 12-21 * * 1-5' + - cron: '15 12-21 * * 1-5' push: branches: - 'main' - - 'michael.richey/dd-test-opt' env: DD_API_KEY: ${{ secrets.DD_API_KEY }} From d2ec8c2d87fcb28b76b69ce788eb5ed9770b4ba6 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 14:58:34 -0400 Subject: [PATCH 29/42] Add the ddtrace option --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 33e28d0f..6accad20 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -48,4 +48,4 @@ jobs: site: datadoghq.com - name: Run tests shell: bash - run: pytest + run: pytest --ddtrace From a770634aae9ad63b507668d54d00ec47969bfac8 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:01:36 -0400 Subject: [PATCH 30/42] Add my branch back --- .github/workflows/datadog_test_optimization.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 6accad20..77807b35 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -12,6 +12,7 @@ on: push: branches: - 'main' + - 'michael.richey/dd-test-opt' env: DD_API_KEY: ${{ secrets.DD_API_KEY }} From c689cb762939040745eafd71215b73daf6e27a55 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:21:17 -0400 Subject: [PATCH 31/42] Try this instead, found in test optimization doc --- .github/workflows/datadog_test_optimization.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 77807b35..3cd55d87 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -40,13 +40,14 @@ jobs: - name: Install requirements run: | python -m pip install . + python -m pip install ddtrace python -m pip install -r scripts/test_requirements.txt - - name: Configure Datadog Test Optimization - uses: datadog/test-visibility-github-action@fa5d4c02a5f2329cf2c62a70fd8ec532317af1c9 - with: - languages: python - api_key: ${{env.DD_API_KEY}} - site: datadoghq.com +# - name: Configure Datadog Test Optimization +# uses: datadog/test-visibility-github-action@fa5d4c02a5f2329cf2c62a70fd8ec532317af1c9 +# with: +# languages: python +# api_key: ${{env.DD_API_KEY}} +# site: datadoghq.com - name: Run tests shell: bash run: pytest --ddtrace From 9b105ef69b698be49b9e6df4e855c9305102949b Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:38:08 -0400 Subject: [PATCH 32/42] Try this way --- .github/workflows/test.yml | 5 +++++ .github/workflows/test_integrations.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f419fb9..085c85c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,11 @@ env: DD_SOURCE_API_KEY: ${{ secrets.DD_SOURCE_API_KEY }} DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} + PYTEST_ADDOPTS: "--ddtrace" + DD_ENV: ci + DD_SERVICE: datadog-sync-cli + DD_TAGS: "team:hamr" + DD_TRACE_ANALYTICS_ENABLED: "true" jobs: test: diff --git a/.github/workflows/test_integrations.yml b/.github/workflows/test_integrations.yml index 2285e772..d2b24d1e 100644 --- a/.github/workflows/test_integrations.yml +++ b/.github/workflows/test_integrations.yml @@ -52,7 +52,7 @@ jobs: DD_AGENT_HOST: localhost DD_ENV: ci DD_SERVICE: datadog-sync-cli - DD_TAGS: "team:integrations-tools-and-libraries" + DD_TAGS: "team:hamr" DD_TRACE_ANALYTICS_ENABLED: "true" RECORD: "none" PYTEST_ADDOPTS: "--ddtrace" From 2ff03aa5aa32713899318b1255d825b521078b71 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:41:23 -0400 Subject: [PATCH 33/42] Try through tox --- .github/workflows/datadog_test_optimization.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 3cd55d87..d057ec6d 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -23,6 +23,11 @@ env: DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} RECORD: false + PYTEST_ADDOPTS: "--ddtrace" + DD_ENV: ci + DD_SERVICE: datadog-sync-cli + DD_TAGS: "team:hamr" + DD_TRACE_ANALYTICS_ENABLED: "true" jobs: test-optimization: @@ -41,13 +46,7 @@ jobs: run: | python -m pip install . python -m pip install ddtrace + python -m pip install tox python -m pip install -r scripts/test_requirements.txt -# - name: Configure Datadog Test Optimization -# uses: datadog/test-visibility-github-action@fa5d4c02a5f2329cf2c62a70fd8ec532317af1c9 -# with: -# languages: python -# api_key: ${{env.DD_API_KEY}} -# site: datadoghq.com - - name: Run tests - shell: bash - run: pytest --ddtrace + - name: Run unit tests + run: tox -e py312 From 7bf5c6bf703b378021872f81cfb74887caa681f8 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:44:04 -0400 Subject: [PATCH 34/42] Remove thes --- .github/workflows/datadog_test_optimization.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index d057ec6d..bce2c81b 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -45,8 +45,6 @@ jobs: - name: Install requirements run: | python -m pip install . - python -m pip install ddtrace python -m pip install tox - python -m pip install -r scripts/test_requirements.txt - name: Run unit tests run: tox -e py312 From a20d372fcb3313c5c139695e137df09f8ddffffe Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:49:03 -0400 Subject: [PATCH 35/42] Try updating ddtrace --- .github/workflows/datadog_test_optimization.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index bce2c81b..dfd43aad 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -27,7 +27,7 @@ env: DD_ENV: ci DD_SERVICE: datadog-sync-cli DD_TAGS: "team:hamr" - DD_TRACE_ANALYTICS_ENABLED: "true" +# DD_TRACE_ANALYTICS_ENABLED: "true" jobs: test-optimization: diff --git a/setup.cfg b/setup.cfg index 73310927..7084239f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,7 @@ console_scripts = [options.extras_require] tests = - ddtrace==2.21.1 + ddtrace>=2.21.1 black==24.3.0 pytest==8.1.1 pytest-black From 9d3275b99a3e6a23e1532df85278ff5a79d397b4 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 15:56:09 -0400 Subject: [PATCH 36/42] Set env var back to true --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index dfd43aad..bce2c81b 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -27,7 +27,7 @@ env: DD_ENV: ci DD_SERVICE: datadog-sync-cli DD_TAGS: "team:hamr" -# DD_TRACE_ANALYTICS_ENABLED: "true" + DD_TRACE_ANALYTICS_ENABLED: "true" jobs: test-optimization: From 944bdaa3f5ed9d262e2441a1d6a09caed61c0479 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 16:09:56 -0400 Subject: [PATCH 37/42] Add the agent for traces --- .github/workflows/datadog_test_optimization.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index bce2c81b..9c121b82 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -31,6 +31,15 @@ env: jobs: test-optimization: + services: + datadog-agent: + image: datadog/agent:latest + ports: + - 8126:8126 + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_HOSTNAME: "none" + DD_INSIDE_CI: "true" strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] From 6a534b432049e53a87e84775fbb7eb0b32e52ae7 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 16:19:07 -0400 Subject: [PATCH 38/42] Move the agent into the os matrix --- .github/workflows/datadog_test_optimization.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 9c121b82..547fe200 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -31,6 +31,10 @@ env: jobs: test-optimization: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} services: datadog-agent: image: datadog/agent:latest @@ -40,10 +44,6 @@ jobs: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_HOSTNAME: "none" DD_INSIDE_CI: "true" - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - runs-on: ${{ matrix.os }} steps: - name: Install Python uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c From 0484e7f0ae1e6171a827770a3eba4b9ad33988ea Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 16:25:01 -0400 Subject: [PATCH 39/42] Back to trying the visibility script --- .github/workflows/datadog_test_optimization.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 547fe200..791f54ee 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -35,15 +35,6 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} - services: - datadog-agent: - image: datadog/agent:latest - ports: - - 8126:8126 - env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} - DD_HOSTNAME: "none" - DD_INSIDE_CI: "true" steps: - name: Install Python uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c @@ -51,6 +42,12 @@ jobs: python-version: 3.12 - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v2 + with: + languages: java + api_key: ${{ secrets.DD_API_KEY }} + site: datadoghq.com - name: Install requirements run: | python -m pip install . From 0cfbd7b5563be1fb3e749a2e0494a1f31123afb4 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Wed, 7 May 2025 16:37:39 -0400 Subject: [PATCH 40/42] Java -> Python for lang --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 791f54ee..a011c097 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -45,7 +45,7 @@ jobs: - name: Configure Datadog Test Optimization uses: datadog/test-visibility-github-action@v2 with: - languages: java + languages: python api_key: ${{ secrets.DD_API_KEY }} site: datadoghq.com - name: Install requirements From 4be2b7d4798ba436df44e82a43e0a6e049f33ea7 Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 8 May 2025 12:37:14 -0400 Subject: [PATCH 41/42] Directly pass in --ddtrace to pytest --- .github/workflows/datadog_test_optimization.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index a011c097..6d5440f6 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -23,7 +23,6 @@ env: DD_SOURCE_APP_KEY: ${{ secrets.DD_SOURCE_APP_KEY }} DD_SOURCE_API_URL: ${{ secrets.DD_SOURCE_API_URL }} RECORD: false - PYTEST_ADDOPTS: "--ddtrace" DD_ENV: ci DD_SERVICE: datadog-sync-cli DD_TAGS: "team:hamr" @@ -53,4 +52,4 @@ jobs: python -m pip install . python -m pip install tox - name: Run unit tests - run: tox -e py312 + run: tox -e py312 --ddtrace From a4261da293738cb10f4fe277896261d5939c249f Mon Sep 17 00:00:00 2001 From: "michael.richey" Date: Thu, 8 May 2025 12:47:28 -0400 Subject: [PATCH 42/42] Pass args into tox differently --- .github/workflows/datadog_test_optimization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datadog_test_optimization.yml b/.github/workflows/datadog_test_optimization.yml index 6d5440f6..f0f257c6 100644 --- a/.github/workflows/datadog_test_optimization.yml +++ b/.github/workflows/datadog_test_optimization.yml @@ -52,4 +52,4 @@ jobs: python -m pip install . python -m pip install tox - name: Run unit tests - run: tox -e py312 --ddtrace + run: tox -e py312 -- --ddtrace