toxgen: Add variants & move OpenAI under toxgen (#4730)

Adds supports for variants, i.e., the same test suite running with a
slightly different setup (for instance, a different set of dependencies,
like `openai` and `openai_notiktoken`).

To add a variant, simply add a new test suite to the config.

The tricky part is naming. I had to rename `openai` to `openai_base`
since otherwise the `openai_notiktoken` and `openai_agents` test suite
would be run with `tox -e py<version>-openai` / `./scripts/runtox.sh
py<version>-openai` due to how tox works. They should be treated as
three different suites.

Closes #4507

Author: sentrivana

.github/workflows/test-integrations-ai.yml

@@ -62,10 +62,14 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-latest"
-      - name: Test openai latest
+      - name: Test openai_base latest
         run: |
           set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest"
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_base-latest"
+      - name: Test openai_notiktoken latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_notiktoken-latest"
       - name: Test openai_agents latest
         run: |
           set -x # print commands that are executed
@@ -141,10 +145,14 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain"
-      - name: Test openai pinned
+      - name: Test openai_base pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_base"
+      - name: Test openai_notiktoken pinned
         run: |
           set -x # print commands that are executed
-          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai"
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_notiktoken"
       - name: Test openai_agents pinned
         run: |
           set -x # print commands that are executed

scripts/populate_tox/README.md

@@ -153,6 +153,14 @@ be expressed like so:
 }
 ```
 
+### `integration_name`
+
+Sometimes, the name of the test suite doesn't match the name of the integration.
+For example, we have the `openai_base` and `openai_notiktoken` test suites, both
+of which are actually testing the `openai` integration. If this is the case, you can use the `integration_name` key to define the name of the integration. If not provided, it will default to the name of the test suite.
+
+Linking an integration to a test suite allows the script to access integration configuration like for example the minimum version defined in `sentry_sdk/integrations/__init__.py`.
+
 
 ## How-Tos
 

scripts/populate_tox/config.py

@@ -139,6 +139,24 @@
     "loguru": {
         "package": "loguru",
     },
+    "openai_base": {
+        "package": "openai",
+        "integration_name": "openai",
+        "deps": {
+            "*": ["pytest-asyncio", "tiktoken"],
+            "<1.55": ["httpx<0.28"],
+        },
+        "python": ">=3.8",
+    },
+    "openai_notiktoken": {
+        "package": "openai",
+        "integration_name": "openai",
+        "deps": {
+            "*": ["pytest-asyncio"],
+            "<1.55": ["httpx<0.28"],
+        },
+        "python": ">=3.8",
+    },
     "openai_agents": {
         "package": "openai-agents",
         "deps": {

scripts/populate_tox/populate_tox.py

@@ -76,8 +76,6 @@
     "httpx",
     "langchain",
     "langchain_notiktoken",
-    "openai",
-    "openai_notiktoken",
     "pure_eval",
     "quart",
     "ray",
@@ -141,7 +139,11 @@ def _prefilter_releases(
     - the list of prefiltered releases
     - an optional prerelease if there is one that should be tested
     """
-    min_supported = _MIN_VERSIONS.get(integration)
+    integration_name = (
+        TEST_SUITE_CONFIG[integration].get("integration_name") or integration
+    )
+
+    min_supported = _MIN_VERSIONS.get(integration_name)
     if min_supported is not None:
         min_supported = Version(".".join(map(str, min_supported)))
     else:

scripts/populate_tox/tox.jinja

@@ -83,13 +83,6 @@ envlist =
     {py3.9,py3.11,py3.12}-langchain-latest
     {py3.9,py3.11,py3.12}-langchain-notiktoken
 
-    # OpenAI
-    {py3.9,py3.11,py3.12}-openai-v1.0
-    {py3.9,py3.11,py3.12}-openai-v1.22
-    {py3.9,py3.11,py3.12}-openai-v1.55
-    {py3.9,py3.11,py3.12}-openai-latest
-    {py3.9,py3.11,py3.12}-openai-notiktoken
-
     # OpenTelemetry (OTel)
     {py3.7,py3.9,py3.12,py3.13}-opentelemetry
 
@@ -252,20 +245,6 @@ deps =
     langchain-{latest,notiktoken}: openai>=1.6.1
     langchain-latest: tiktoken~=0.6.0
 
-    # OpenAI
-    openai: pytest-asyncio
-    openai-v1.0: openai~=1.0.0
-    openai-v1.0: tiktoken
-    openai-v1.0: httpx<0.28.0
-    openai-v1.22: openai~=1.22.0
-    openai-v1.22: tiktoken
-    openai-v1.22: httpx<0.28.0
-    openai-v1.55: openai~=1.55.0
-    openai-v1.55: tiktoken
-    openai-latest: openai
-    openai-latest: tiktoken~=0.6.0
-    openai-notiktoken: openai
-
     # OpenTelemetry (OTel)
     opentelemetry: opentelemetry-distro
 
@@ -401,7 +380,8 @@ setenv =
     launchdarkly: TESTPATH=tests/integrations/launchdarkly
     litestar: TESTPATH=tests/integrations/litestar
     loguru: TESTPATH=tests/integrations/loguru
-    openai: TESTPATH=tests/integrations/openai
+    openai_base: TESTPATH=tests/integrations/openai
+    openai_notiktoken: TESTPATH=tests/integrations/openai
     openai_agents: TESTPATH=tests/integrations/openai_agents
     openfeature: TESTPATH=tests/integrations/openfeature
     opentelemetry: TESTPATH=tests/integrations/opentelemetry

scripts/split_tox_gh_actions/split_tox_gh_actions.py

@@ -62,7 +62,8 @@
         "anthropic",
         "cohere",
         "langchain",
-        "openai",
+        "openai_base",
+        "openai_notiktoken",
         "openai_agents",
         "huggingface_hub",
     ],

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-09-01T12:08:33.833560+00:00
+# Last generated: 2025-09-01T14:09:50.564158+00:00
 
 [tox]
 requires =
@@ -83,13 +83,6 @@ envlist =
     {py3.9,py3.11,py3.12}-langchain-latest
     {py3.9,py3.11,py3.12}-langchain-notiktoken
 
-    # OpenAI
-    {py3.9,py3.11,py3.12}-openai-v1.0
-    {py3.9,py3.11,py3.12}-openai-v1.22
-    {py3.9,py3.11,py3.12}-openai-v1.55
-    {py3.9,py3.11,py3.12}-openai-latest
-    {py3.9,py3.11,py3.12}-openai-notiktoken
-
     # OpenTelemetry (OTel)
     {py3.7,py3.9,py3.12,py3.13}-opentelemetry
 
@@ -145,6 +138,16 @@ envlist =
     {py3.9,py3.11,py3.12}-cohere-v5.13.12
     {py3.9,py3.11,py3.12}-cohere-v5.17.0
 
+    {py3.8,py3.11,py3.12}-openai_base-v1.0.1
+    {py3.8,py3.11,py3.12}-openai_base-v1.35.15
+    {py3.8,py3.11,py3.12}-openai_base-v1.69.0
+    {py3.8,py3.12,py3.13}-openai_base-v1.102.0
+
+    {py3.8,py3.11,py3.12}-openai_notiktoken-v1.0.1
+    {py3.8,py3.11,py3.12}-openai_notiktoken-v1.35.15
+    {py3.8,py3.11,py3.12}-openai_notiktoken-v1.69.0
+    {py3.8,py3.12,py3.13}-openai_notiktoken-v1.102.0
+
     {py3.10,py3.11,py3.12}-openai_agents-v0.0.19
     {py3.10,py3.12,py3.13}-openai_agents-v0.1.0
     {py3.10,py3.12,py3.13}-openai_agents-v0.2.10
@@ -423,20 +426,6 @@ deps =
     langchain-{latest,notiktoken}: openai>=1.6.1
     langchain-latest: tiktoken~=0.6.0
 
-    # OpenAI
-    openai: pytest-asyncio
-    openai-v1.0: openai~=1.0.0
-    openai-v1.0: tiktoken
-    openai-v1.0: httpx<0.28.0
-    openai-v1.22: openai~=1.22.0
-    openai-v1.22: tiktoken
-    openai-v1.22: httpx<0.28.0
-    openai-v1.55: openai~=1.55.0
-    openai-v1.55: tiktoken
-    openai-latest: openai
-    openai-latest: tiktoken~=0.6.0
-    openai-notiktoken: openai
-
     # OpenTelemetry (OTel)
     opentelemetry: opentelemetry-distro
 
@@ -527,6 +516,23 @@ deps =
     cohere-v5.13.12: cohere==5.13.12
     cohere-v5.17.0: cohere==5.17.0
 
+    openai_base-v1.0.1: openai==1.0.1
+    openai_base-v1.35.15: openai==1.35.15
+    openai_base-v1.69.0: openai==1.69.0
+    openai_base-v1.102.0: openai==1.102.0
+    openai_base: pytest-asyncio
+    openai_base: tiktoken
+    openai_base-v1.0.1: httpx<0.28
+    openai_base-v1.35.15: httpx<0.28
+
+    openai_notiktoken-v1.0.1: openai==1.0.1
+    openai_notiktoken-v1.35.15: openai==1.35.15
+    openai_notiktoken-v1.69.0: openai==1.69.0
+    openai_notiktoken-v1.102.0: openai==1.102.0
+    openai_notiktoken: pytest-asyncio
+    openai_notiktoken-v1.0.1: httpx<0.28
+    openai_notiktoken-v1.35.15: httpx<0.28
+
     openai_agents-v0.0.19: openai-agents==0.0.19
     openai_agents-v0.1.0: openai-agents==0.1.0
     openai_agents-v0.2.10: openai-agents==0.2.10
@@ -831,7 +837,8 @@ setenv =
     launchdarkly: TESTPATH=tests/integrations/launchdarkly
     litestar: TESTPATH=tests/integrations/litestar
     loguru: TESTPATH=tests/integrations/loguru
-    openai: TESTPATH=tests/integrations/openai
+    openai_base: TESTPATH=tests/integrations/openai
+    openai_notiktoken: TESTPATH=tests/integrations/openai
     openai_agents: TESTPATH=tests/integrations/openai_agents
     openfeature: TESTPATH=tests/integrations/openfeature
     opentelemetry: TESTPATH=tests/integrations/opentelemetry