diff --git a/.github/workflows/core_contrib_test_0.yml b/.github/workflows/core_contrib_test_0.yml index 33320b5ef5..40fc729620 100644 --- a/.github/workflows/core_contrib_test_0.yml +++ b/.github/workflows/core_contrib_test_0.yml @@ -2453,6 +2453,36 @@ jobs: - name: Run tests run: tox -e py39-test-util-http -- -ra + py39-test-exporter-credential-provider-gcp: + name: exporter-credential-provider-gcp + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout contrib repo @ SHA - ${{ env.CONTRIB_REPO_SHA }} + uses: actions/checkout@v4 + with: + repository: open-telemetry/opentelemetry-python-contrib + ref: ${{ env.CONTRIB_REPO_SHA }} + + - name: Checkout core repo @ SHA - ${{ env.CORE_REPO_SHA }} + uses: actions/checkout@v4 + with: + repository: open-telemetry/opentelemetry-python + ref: ${{ env.CORE_REPO_SHA }} + path: opentelemetry-python + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py39-test-exporter-credential-provider-gcp -- -ra + py39-test-util-genai: name: util-genai runs-on: ubuntu-latest diff --git a/.github/workflows/lint_0.yml b/.github/workflows/lint_0.yml index 9875fed3f4..309705ae5a 100644 --- a/.github/workflows/lint_0.yml +++ b/.github/workflows/lint_0.yml @@ -1077,6 +1077,25 @@ jobs: - name: Run tests run: tox -e lint-util-http + lint-exporter-credential-provider-gcp: + name: exporter-credential-provider-gcp + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e lint-exporter-credential-provider-gcp + lint-util-genai: name: util-genai runs-on: ubuntu-latest diff --git a/.github/workflows/test_1.yml b/.github/workflows/test_1.yml index 058db6d49b..e60b3c530b 100644 --- a/.github/workflows/test_1.yml +++ b/.github/workflows/test_1.yml @@ -3965,6 +3965,101 @@ jobs: - name: Run tests run: tox -e pypy3-test-util-http -- -ra + py39-test-exporter-credential-provider-gcp_ubuntu-latest: + name: exporter-credential-provider-gcp 3.9 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py39-test-exporter-credential-provider-gcp -- -ra + + py310-test-exporter-credential-provider-gcp_ubuntu-latest: + name: exporter-credential-provider-gcp 3.10 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py310-test-exporter-credential-provider-gcp -- -ra + + py311-test-exporter-credential-provider-gcp_ubuntu-latest: + name: exporter-credential-provider-gcp 3.11 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py311-test-exporter-credential-provider-gcp -- -ra + + py312-test-exporter-credential-provider-gcp_ubuntu-latest: + name: exporter-credential-provider-gcp 3.12 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py312-test-exporter-credential-provider-gcp -- -ra + + py313-test-exporter-credential-provider-gcp_ubuntu-latest: + name: exporter-credential-provider-gcp 3.13 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py313-test-exporter-credential-provider-gcp -- -ra + py39-test-util-genai_ubuntu-latest: name: util-genai 3.9 Ubuntu runs-on: ubuntu-latest @@ -4686,98 +4781,3 @@ jobs: - name: Run tests run: tox -e py39-test-instrumentation-aio-pika-1 -- -ra - - py39-test-instrumentation-aio-pika-2_ubuntu-latest: - name: instrumentation-aio-pika-2 3.9 Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py39-test-instrumentation-aio-pika-2 -- -ra - - py39-test-instrumentation-aio-pika-3_ubuntu-latest: - name: instrumentation-aio-pika-3 3.9 Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py39-test-instrumentation-aio-pika-3 -- -ra - - py310-test-instrumentation-aio-pika-0_ubuntu-latest: - name: instrumentation-aio-pika-0 3.10 Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py310-test-instrumentation-aio-pika-0 -- -ra - - py310-test-instrumentation-aio-pika-1_ubuntu-latest: - name: instrumentation-aio-pika-1 3.10 Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py310-test-instrumentation-aio-pika-1 -- -ra - - py310-test-instrumentation-aio-pika-2_ubuntu-latest: - name: instrumentation-aio-pika-2 3.10 Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py310-test-instrumentation-aio-pika-2 -- -ra diff --git a/.github/workflows/test_2.yml b/.github/workflows/test_2.yml index f352a83d53..8a385e54f4 100644 --- a/.github/workflows/test_2.yml +++ b/.github/workflows/test_2.yml @@ -32,6 +32,101 @@ env: jobs: + py39-test-instrumentation-aio-pika-2_ubuntu-latest: + name: instrumentation-aio-pika-2 3.9 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py39-test-instrumentation-aio-pika-2 -- -ra + + py39-test-instrumentation-aio-pika-3_ubuntu-latest: + name: instrumentation-aio-pika-3 3.9 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py39-test-instrumentation-aio-pika-3 -- -ra + + py310-test-instrumentation-aio-pika-0_ubuntu-latest: + name: instrumentation-aio-pika-0 3.10 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py310-test-instrumentation-aio-pika-0 -- -ra + + py310-test-instrumentation-aio-pika-1_ubuntu-latest: + name: instrumentation-aio-pika-1 3.10 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py310-test-instrumentation-aio-pika-1 -- -ra + + py310-test-instrumentation-aio-pika-2_ubuntu-latest: + name: instrumentation-aio-pika-2 3.10 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py310-test-instrumentation-aio-pika-2 -- -ra + py310-test-instrumentation-aio-pika-3_ubuntu-latest: name: instrumentation-aio-pika-3 3.10 Ubuntu runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b63232109b..8349f4b942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `rstcheck` to pre-commit to stop introducing invalid RST ([#3777](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3777)) +- `opentelemetry-exporter-credential-provider-gcp`: create this package which provides support for supplying your machine's Application Default Credentials (https://cloud.google.com/docs/authentication/application-default-credentials) to the OTLP Exporters created automatically by OpenTelemetry Python's auto instrumentation. These credentials authorize OTLP traces to be sent to `telemetry.googleapis.com`. +[#3766](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3766). ## Version 1.37.0/0.58b0 (2025-09-11) diff --git a/RELEASING.md b/RELEASING.md index 053b235228..41b3491a9b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -92,6 +92,7 @@ The workflow will create a pull request that should be merged in order to procee > - opentelemetry-instrumentation-langchain > - opentelemetry-instrumentation-weaviate > - opentelemetry-util-genai +> - opentelemetry-exporter-credential-provider-gcp > > These libraries are also excluded from the general patch release. diff --git a/eachdist.ini b/eachdist.ini index 72a9bdf63d..0e731e5fd2 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -32,6 +32,7 @@ packages= opentelemetry-exporter-jaeger-proto-grpc opentelemetry-exporter-jaeger opentelemetry-api + opentelemetry-exporter-credential-provider-gcp [prerelease] version=0.59b0.dev diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/LICENSE b/exporter/opentelemetry-exporter-credential-provider-gcp/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/README.rst b/exporter/opentelemetry-exporter-credential-provider-gcp/README.rst new file mode 100644 index 0000000000..e2486f6c57 --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/README.rst @@ -0,0 +1,51 @@ +OpenTelemetry GCP Credential Provider for OTLP Exporters +======================================================== + +.. image:: https://badge.fury.io/py/opentelemetry-exporter-credential-provider-gcp.svg + :target: https://badge.fury.io/py/opentelemetry-exporter-credential-provider-gcp + +.. image:: https://readthedocs.org/projects/google-cloud-opentelemetry/badge/?version=latest + :target: https://google-cloud-opentelemetry.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +This library provides support for supplying your machine's Application Default Credentials (https://cloud.google.com/docs/authentication/application-default-credentials) +to the OTLP Exporters created automatically by OpenTelemetry Python's auto instrumentation. +These credentials authorize OTLP traces to be sent to `telemetry.googleapis.com`. + +Currently `telemetry.googleapis.com` only supports OTLP traces, but eventually the endpoint will +support OTLP logs and metrics too. + +To learn more about instrumentation and observability, including opinionated recommendations +for Google Cloud Observability, visit `Instrumentation and observability +`_. + +For how to export trace data when you use manual instrumentation visit `Migrate from the Trace exporter to the OTLP endpoint +`_. + +Installation +------------ + +.. code:: bash + + pip install opentelemetry-exporter-credential-provider-gcp + +Usage +----- + +Your installed HTTP/GRPC OTLP Exporter must be at release `>=1.37` for this feature. + +Set the following environment variables: +`export OTEL_RESOURCE_ATTRIBUTES="gcp.project_id="` +`export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://telemetry.googleapis.com:443/v1/traces"` + +If you plan to have python auto instrumentation use the GRPC OTLP Exporter to send traces to Cloud Trace: +`export OTEL_PYTHON_EXPORTER_OTLP_GRPC_TRACES_CREDENTIAL_PROVIDER=gcp_grpc_credentials` + +If you plan to have python auto instrumentation use the HTTP OTLP Exporter to send traces to Cloud Trace: +`export OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER=gcp_http_credentials` + + +References +---------- + +* `OpenTelemetry Project `_ diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/pyproject.toml b/exporter/opentelemetry-exporter-credential-provider-gcp/pyproject.toml new file mode 100644 index 0000000000..49710b7f5d --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-exporter-credential-provider-gcp" +dynamic = ["version"] +description = "GCP OTLP Exporter Credential Provider for OpenTelemetry" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "grpcio >= 1.67.0", + "google-auth >= 2.25", + "requests >= 2.7", +] + +[project.entry-points.opentelemetry_otlp_credential_provider] +gcp_grpc_credentials = "opentelemetry.gcp_credential_provider:create_google_grpc_credentials" +gcp_http_credentials = "opentelemetry.gcp_credential_provider:create_google_authorized_session" + +[project.urls] +Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/exporter/opentelemetry-exporter-credential-provider-gcp" +Repository = "https://github.com/open-telemetry/opentelemetry-python-contrib" + +[tool.hatch.version] +path = "src/opentelemetry/gcp_credential_provider/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/__init__.py b/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/__init__.py new file mode 100644 index 0000000000..4905912adf --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/__init__.py @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import grpc +import requests +from google.auth.transport.grpc import AuthMetadataPlugin +from google.auth.transport.requests import AuthorizedSession, Request + +# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false + + +def create_google_grpc_credentials() -> grpc.ChannelCredentials: + credentials, _ = google.auth.default() + return grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials( + AuthMetadataPlugin(credentials=credentials, request=Request()) + ), + ) + + +def create_google_authorized_session() -> requests.Session: + credentials, _ = google.auth.default() + return AuthorizedSession(credentials) diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/version.py b/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/version.py new file mode 100644 index 0000000000..46aee9202b --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/src/opentelemetry/gcp_credential_provider/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.59b0.dev" diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/test-requirements.txt b/exporter/opentelemetry-exporter-credential-provider-gcp/test-requirements.txt new file mode 100644 index 0000000000..04f291e885 --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/test-requirements.txt @@ -0,0 +1,4 @@ +pytest==7.4.4 +opentelemetry-exporter-otlp-proto-http==1.37 +opentelemetry-exporter-otlp-proto-common==1.37 +opentelemetry-proto==1.37 \ No newline at end of file diff --git a/exporter/opentelemetry-exporter-credential-provider-gcp/tests/test_credential_provider.py b/exporter/opentelemetry-exporter-credential-provider-gcp/tests/test_credential_provider.py new file mode 100644 index 0000000000..f2d59e6625 --- /dev/null +++ b/exporter/opentelemetry-exporter-credential-provider-gcp/tests/test_credential_provider.py @@ -0,0 +1,51 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from os import environ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from google.auth.transport.requests import AuthorizedSession + +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.sdk.environment_variables import ( + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_TRACES_CREDENTIAL_PROVIDER, + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER, +) + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportMissingImports=false, reportUntypedFunctionDecorator=false + + +class TestOTLPTraceAutoInstrumentGcpCredential(TestCase): + @patch("google.auth.default") + @patch.dict( + environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER: "gcp_http_credentials", + _OTEL_PYTHON_EXPORTER_OTLP_GRPC_TRACES_CREDENTIAL_PROVIDER: "gcp_grpc_credentials", + }, + ) + def test_loads_otlp_exporters_with_google_creds( + self, mock_default: MagicMock + ): # pylint: disable=no-self-use + """Test that OTel configuration internals can load the credentials from entrypoint by + name""" + mock_credentials = MagicMock() + mock_credentials.project_id = "test-project" + mock_default.return_value = (mock_credentials, "test-project") + http_exporter = OTLPSpanExporter() + assert isinstance(http_exporter._session, AuthorizedSession) + # Assert that google.auth.default was called + mock_default.assert_called_once() diff --git a/pyproject.toml b/pyproject.toml index 822379765d..30ada5bb25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,6 +205,7 @@ include = [ "instrumentation-genai/opentelemetry-instrumentation-langchain", "instrumentation-genai/opentelemetry-instrumentation-weaviate", "util/opentelemetry-util-genai", + "exporter/opentelemetry-exporter-credential-provider-gcp", ] # We should also add type hints to the test suite - It helps on finding bugs. # We are excluding for now because it's easier, and more important to add to the instrumentation packages. diff --git a/tox.ini b/tox.ini index 854c3e7884..4004b554d5 100644 --- a/tox.ini +++ b/tox.ini @@ -356,6 +356,10 @@ envlist = pypy3-test-util-http lint-util-http + ; opentelemetry-exporter-credential-provider-gcp + py3{9,10,11,12,13}-test-exporter-credential-provider-gcp + lint-exporter-credential-provider-gcp + ; opentelemetry-util-genai py3{9,10,11,12,13}-test-util-genai pypy3-test-util-genai @@ -438,7 +442,6 @@ test_deps = opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils - deps = lint: -r dev-requirements.txt coverage: pytest @@ -721,6 +724,9 @@ deps = util-http: -r {toxinidir}/util/opentelemetry-util-http/test-requirements.txt util-http: {toxinidir}/util/opentelemetry-util-http + exporter-credential-provider-gcp: -r {toxinidir}/exporter/opentelemetry-exporter-credential-provider-gcp/test-requirements.txt + exporter-credential-provider-gcp: {toxinidir}/exporter/opentelemetry-exporter-credential-provider-gcp + util-genai: {[testenv]test_deps} util-genai: -r {toxinidir}/util/opentelemetry-util-genai/test-requirements.txt util-genai: {toxinidir}/util/opentelemetry-util-genai @@ -920,6 +926,9 @@ commands = test-util-http: pytest {toxinidir}/util/opentelemetry-util-http/tests {posargs} lint-util-http: sh -c "cd util && pylint --rcfile ../.pylintrc opentelemetry-util-http" + test-exporter-credential-provider-gcp: pytest {toxinidir}/exporter/opentelemetry-exporter-credential-provider-gcp/tests {posargs} + lint-exporter-credential-provider-gcp: sh -c "cd exporter && pylint --rcfile ../.pylintrc opentelemetry-exporter-credential-provider-gcp" + test-util-genai: pytest {toxinidir}/util/opentelemetry-util-genai/tests {posargs} lint-util-genai: sh -c "cd util && pylint --rcfile ../.pylintrc opentelemetry-util-genai" @@ -960,6 +969,7 @@ deps = {toxinidir}/opentelemetry-instrumentation {toxinidir}/util/opentelemetry-util-http {toxinidir}/util/opentelemetry-util-genai + {toxinidir}/exporter/opentelemetry-exporter-credential-provider-gcp changedir = docs @@ -1067,6 +1077,7 @@ deps = {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-google-genai[instruments] {toxinidir}/instrumentation/opentelemetry-instrumentation-aiokafka[instruments] {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncclick[instruments] + {toxinidir}/exporter/opentelemetry-exporter-credential-provider-gcp commands = pyright diff --git a/util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/upload_hook.py b/util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/upload_hook.py deleted file mode 100644 index 9180b98eb8..0000000000 --- a/util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/upload_hook.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module defines the generic hooks for GenAI content uploading - -The hooks are specified as part of semconv in `Uploading content to external storage -`__. - -This module defines the `UploadHook` type that custom implementations should implement, and a -`load_upload_hook` function to load it from an entry point. -""" - -from __future__ import annotations - -import logging -from os import environ -from typing import Any, Protocol, cast, runtime_checkable - -from opentelemetry._logs import LogRecord -from opentelemetry.trace import Span -from opentelemetry.util._importlib_metadata import ( - entry_points, # pyright: ignore[reportUnknownVariableType] -) -from opentelemetry.util.genai import types -from opentelemetry.util.genai.environment_variables import ( - OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK, -) - -_logger = logging.getLogger(__name__) - - -@runtime_checkable -class UploadHook(Protocol): - """A hook to upload GenAI content to an external storage. - - This is the interface for a hook that can be - used to upload GenAI content to an external storage. The hook is a - callable that takes the inputs, outputs, and system instruction of a - GenAI interaction, as well as the span and log record associated with - it. - - The hook can be used to upload the content to any external storage, - such as a database, a file system, or a cloud storage service. - - The span and log_record arguments should be provided based on the content capturing mode - :func:`~opentelemetry.util.genai.utils.get_content_capturing_mode`. - - Args: - inputs: The inputs of the GenAI interaction. - outputs: The outputs of the GenAI interaction. - system_instruction: The system instruction of the GenAI - interaction. - span: The span associated with the GenAI interaction. - log_record: The event log associated with the GenAI - interaction. - """ - - def upload( - self, - *, - inputs: list[types.InputMessage], - outputs: list[types.OutputMessage], - system_instruction: list[types.MessagePart], - span: Span | None = None, - log_record: LogRecord | None = None, - ) -> None: ... - - -class _NoOpUploadHook(UploadHook): - def upload(self, **kwargs: Any) -> None: - return None - - -def load_upload_hook() -> UploadHook: - """Load the upload hook from entry point or return a noop implementation - - This function loads an upload hook from the entry point group - ``opentelemetry_genai_upload_hook`` with name coming from - :envvar:`OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK`. If one can't be found, returns a no-op - implementation. - """ - hook_name = environ.get(OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK, None) - if not hook_name: - return _NoOpUploadHook() - - for entry_point in entry_points(group="opentelemetry_genai_upload_hook"): # pyright: ignore[reportUnknownVariableType] - name = cast(str, entry_point.name) # pyright: ignore[reportUnknownMemberType] - try: - if hook_name != name: - continue - - hook = entry_point.load()() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] - if not isinstance(hook, UploadHook): - _logger.debug("%s is not a valid UploadHook. Using noop", name) - continue - - _logger.debug("Using UploadHook %s", name) - return hook - - except Exception: # pylint: disable=broad-except - _logger.exception( - "UploadHook %s configuration failed. Using noop", name - ) - - return _NoOpUploadHook() - - -__all__ = ["UploadHook", "load_upload_hook"] diff --git a/util/opentelemetry-util-genai-dev/tests/test_upload_hook.py b/util/opentelemetry-util-genai-dev/tests/test_upload_hook.py deleted file mode 100644 index 93731bce95..0000000000 --- a/util/opentelemetry-util-genai-dev/tests/test_upload_hook.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from dataclasses import dataclass -from typing import Any, Callable -from unittest import TestCase -from unittest.mock import Mock, patch - -from opentelemetry.util.genai.environment_variables import ( - OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK, -) -from opentelemetry.util.genai.upload_hook import ( - UploadHook, - _NoOpUploadHook, - load_upload_hook, -) - - -class FakeUploadHook(UploadHook): - def upload(self, **kwargs: Any): - pass - - -class InvalidUploadHook: - pass - - -@dataclass -class FakeEntryPoint: - name: str - load: Callable[[], type[UploadHook]] - - -class TestUploadHook(TestCase): - @patch.dict("os.environ", {}) - def test_load_upload_hook_noop(self): - self.assertIsInstance(load_upload_hook(), _NoOpUploadHook) - - @patch( - "opentelemetry.util.genai.upload_hook.entry_points", - ) - @patch.dict( - "os.environ", {OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK: "my-hook"} - ) - def test_load_upload_hook_custom(self, mock_entry_points: Mock): - mock_entry_points.return_value = [ - FakeEntryPoint("my-hook", lambda: FakeUploadHook) - ] - - self.assertIsInstance(load_upload_hook(), FakeUploadHook) - - @patch("opentelemetry.util.genai.upload_hook.entry_points") - @patch.dict( - "os.environ", {OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK: "my-hook"} - ) - def test_load_upload_hook_invalid(self, mock_entry_points: Mock): - mock_entry_points.return_value = [ - FakeEntryPoint("my-hook", lambda: InvalidUploadHook) - ] - - with self.assertLogs(level=logging.DEBUG) as logs: - self.assertIsInstance(load_upload_hook(), _NoOpUploadHook) - self.assertEqual(len(logs.output), 1) - self.assertIn("is not a valid UploadHook. Using noop", logs.output[0]) - - @patch("opentelemetry.util.genai.upload_hook.entry_points") - @patch.dict( - "os.environ", {OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK: "my-hook"} - ) - def test_load_upload_hook_error(self, mock_entry_points: Mock): - def load(): - raise RuntimeError("error") - - mock_entry_points.return_value = [FakeEntryPoint("my-hook", load)] - - self.assertIsInstance(load_upload_hook(), _NoOpUploadHook) - - @patch("opentelemetry.util.genai.upload_hook.entry_points") - @patch.dict( - "os.environ", {OTEL_INSTRUMENTATION_GENAI_UPLOAD_HOOK: "my-hook"} - ) - def test_load_upload_hook_not_found(self, mock_entry_points: Mock): - mock_entry_points.return_value = [ - FakeEntryPoint("other-hook", lambda: FakeUploadHook) - ] - - self.assertIsInstance(load_upload_hook(), _NoOpUploadHook) diff --git a/util/opentelemetry-util-genai/CHANGELOG.md b/util/opentelemetry-util-genai/CHANGELOG.md index 24877552f5..b1f7592c75 100644 --- a/util/opentelemetry-util-genai/CHANGELOG.md +++ b/util/opentelemetry-util-genai/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Add jsonlines support to fsspec uploader + ([https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3791](#3791)) + ## Version 0.1b0 (2025-09-24) - Add completion hook to genai utils to implement semconv v1.37. diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fsspec_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fsspec_upload/completion_hook.py index 56d7b0dcd6..1d7cc3f3c0 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fsspec_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fsspec_upload/completion_hook.py @@ -24,6 +24,7 @@ from contextlib import ExitStack from dataclasses import asdict, dataclass from functools import partial +from os import environ from time import time from typing import Any, Callable, Final, Literal, TextIO, cast from uuid import uuid4 @@ -35,6 +36,9 @@ from opentelemetry.trace import Span from opentelemetry.util.genai import types from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.environment_variables import ( + OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, +) GEN_AI_INPUT_MESSAGES_REF: Final = ( gen_ai_attributes.GEN_AI_INPUT_MESSAGES + "_ref" @@ -46,6 +50,10 @@ gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + "_ref" ) +_MESSAGE_INDEX_KEY = "index" + +Format = Literal["json", "jsonl"] +_FORMATS: tuple[Format, ...] = ("json", "jsonl") _logger = logging.getLogger(__name__) @@ -94,10 +102,27 @@ def __init__( *, base_path: str, max_size: int = 20, + upload_format: Format | None = None, ) -> None: self._base_path = base_path self._max_size = max_size + if upload_format not in _FORMATS + (None,): + raise ValueError( + f"Invalid {upload_format=}. Must be one of {_FORMATS}" + ) + + if upload_format is None: + environ_format = environ.get( + OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, "json" + ).lower() + if environ_format not in _FORMATS: + upload_format = "json" + else: + upload_format = environ_format + + self._format: Final[Literal["json", "jsonl"]] = upload_format + # Use a ThreadPoolExecutor for its queueing and thread management. The semaphore # limits the number of queued tasks. If the queue is full, data will be dropped. self._executor = ThreadPoolExecutor(max_workers=max_size) @@ -139,27 +164,39 @@ def _calculate_ref_path(self) -> CompletionRefs: uuid_str = str(uuid4()) return CompletionRefs( inputs_ref=posixpath.join( - self._base_path, f"{uuid_str}_inputs.json" + self._base_path, f"{uuid_str}_inputs.{self._format}" ), outputs_ref=posixpath.join( - self._base_path, f"{uuid_str}_outputs.json" + self._base_path, f"{uuid_str}_outputs.{self._format}" ), system_instruction_ref=posixpath.join( - self._base_path, f"{uuid_str}_system_instruction.json" + self._base_path, + f"{uuid_str}_system_instruction.{self._format}", ), ) - @staticmethod def _do_upload( - path: str, json_encodeable: Callable[[], JsonEncodeable] + self, path: str, json_encodeable: Callable[[], JsonEncodeable] ) -> None: + if self._format == "json": + # output as a single line with the json messages array + message_lines = [json_encodeable()] + else: + # output as one line per message in the array + message_lines = json_encodeable() + # add an index for streaming readers of jsonl + for message_idx, line in enumerate(message_lines): + line[_MESSAGE_INDEX_KEY] = message_idx + with fsspec_open(path, "w") as file: - json.dump( - json_encodeable(), - file, - separators=(",", ":"), - cls=Base64JsonEncoder, - ) + for message in message_lines: + json.dump( + message, + file, + separators=(",", ":"), + cls=Base64JsonEncoder, + ) + file.write("\n") def on_completion( self, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/environment_variables.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/environment_variables.py index 0ff089d82a..7c03169743 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/environment_variables.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/environment_variables.py @@ -43,3 +43,13 @@ `_ for advanced use cases. """ + +OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT = ( + "OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT" +) +""" +.. envvar:: OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT + +The format to use when uploading prompt and response data. Must be one of ``json`` or +``jsonl``. Defaults to ``json``. +""" diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/span_utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/span_utils.py index 95c5936af2..723d6bdccb 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/span_utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/span_utils.py @@ -49,7 +49,7 @@ def _apply_common_span_attributes( request_model = invocation.request_model provider = invocation.provider span.update_name( - f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}" + f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}".strip() ) span.set_attribute( GenAI.GEN_AI_OPERATION_NAME, GenAI.GenAiOperationNameValues.CHAT.value @@ -72,11 +72,11 @@ def _apply_common_span_attributes( ) if invocation.response_id is not None: span.set_attribute(GenAI.GEN_AI_RESPONSE_ID, invocation.response_id) - if isinstance(invocation.input_tokens, (int, float)): + if invocation.input_tokens is not None: span.set_attribute( GenAI.GEN_AI_USAGE_INPUT_TOKENS, invocation.input_tokens ) - if isinstance(invocation.output_tokens, (int, float)): + if invocation.output_tokens is not None: span.set_attribute( GenAI.GEN_AI_USAGE_OUTPUT_TOKENS, invocation.output_tokens ) @@ -104,7 +104,7 @@ def _maybe_set_span_messages( ) -def _maybe_set_span_extra_attributes( +def _set_span_extra_attributes( span: Span, attributes: Dict[str, Any], ) -> None: @@ -118,7 +118,7 @@ def _apply_finish_attributes(span: Span, invocation: LLMInvocation) -> None: _maybe_set_span_messages( span, invocation.input_messages, invocation.output_messages ) - _maybe_set_span_extra_attributes(span, invocation.attributes) + _set_span_extra_attributes(span, invocation.attributes) def _apply_error_attributes(span: Span, error: Error) -> None: diff --git a/util/opentelemetry-util-genai/tests/test_fsspec_upload.py b/util/opentelemetry-util-genai/tests/test_fsspec_upload.py index 96c76d8458..2d390adfff 100644 --- a/util/opentelemetry-util-genai/tests/test_fsspec_upload.py +++ b/util/opentelemetry-util-genai/tests/test_fsspec_upload.py @@ -19,7 +19,6 @@ import sys import threading from contextlib import contextmanager -from dataclasses import asdict from typing import Any from unittest import TestCase from unittest.mock import MagicMock, patch @@ -76,6 +75,24 @@ def test_fsspec_entry_point_no_fsspec(self): role="user", parts=[types.Text(content="What is the capital of France?")], ), + types.InputMessage( + role="assistant", + parts=[ + types.ToolCall( + id="get_capital_0", + name="get_capital", + arguments={"city": "Paris"}, + ) + ], + ), + types.InputMessage( + role="user", + parts=[ + types.ToolCallResponse( + id="get_capital_0", response={"capital": "Paris"} + ) + ], + ), ] FAKE_OUTPUTS = [ types.OutputMessage( @@ -197,6 +214,44 @@ def test_failed_upload_logs(self): self.assertIn("fsspec uploader failed", logs.output[0]) + def test_invalid_upload_format(self): + with self.assertRaisesRegex(ValueError, "Invalid upload_format"): + FsspecUploadCompletionHook( + base_path=BASE_PATH, upload_format="invalid" + ) + + def test_parse_upload_format_envvar(self): + for envvar_value, expect in ( + ("", "json"), + ("json", "json"), + ("invalid", "json"), + ("jsonl", "jsonl"), + ("jSoNl", "jsonl"), + ): + with patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT": envvar_value}, + clear=True, + ): + hook = FsspecUploadCompletionHook(base_path=BASE_PATH) + self.addCleanup(hook.shutdown) + self.assertEqual( + hook._format, + expect, + f"expected upload format {expect=} with {envvar_value=} got {hook._format}", + ) + + with patch.dict( + "os.environ", + {"OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT": "json"}, + clear=True, + ): + hook = FsspecUploadCompletionHook( + base_path=BASE_PATH, upload_format="jsonl" + ) + self.addCleanup(hook.shutdown) + self.assertEqual(hook._format, "jsonl") + def test_upload_after_shutdown_logs(self): self.hook.shutdown() with self.assertLogs(level=logging.INFO) as logs: @@ -212,25 +267,15 @@ def test_upload_after_shutdown_logs(self): ) -class FsspecUploaderTest(TestCase): - def test_upload(self): - FsspecUploadCompletionHook._do_upload( - "memory://my_path", - lambda: [asdict(fake_input) for fake_input in FAKE_INPUTS], - ) - - with fsspec.open("memory://my_path", "r") as file: - self.assertEqual( - file.read(), - '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"}]}]', - ) - - class TestFsspecUploadCompletionHookIntegration(TestBase): def setUp(self): super().setUp() self.hook = FsspecUploadCompletionHook(base_path=BASE_PATH) + def create_hook(self) -> FsspecUploadCompletionHook: + self.hook = FsspecUploadCompletionHook(base_path=BASE_PATH) + return self.hook + def tearDown(self): super().tearDown() self.hook.shutdown() @@ -271,15 +316,15 @@ def test_upload_completions(self): self.assert_fsspec_equal( span.attributes["gen_ai.input.messages_ref"], - '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"}]}]', + '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"}]},{"role":"assistant","parts":[{"arguments":{"city":"Paris"},"name":"get_capital","id":"get_capital_0","type":"tool_call"}]},{"role":"user","parts":[{"response":{"capital":"Paris"},"id":"get_capital_0","type":"tool_call_response"}]}]\n', ) self.assert_fsspec_equal( span.attributes["gen_ai.output.messages_ref"], - '[{"role":"assistant","parts":[{"content":"Paris","type":"text"}],"finish_reason":"stop"}]', + '[{"role":"assistant","parts":[{"content":"Paris","type":"text"}],"finish_reason":"stop"}]\n', ) self.assert_fsspec_equal( span.attributes["gen_ai.system_instructions_ref"], - '[{"content":"You are a helpful assistant.","type":"text"}]', + '[{"content":"You are a helpful assistant.","type":"text"}]\n', ) def test_stamps_empty_log(self): @@ -316,5 +361,59 @@ def test_upload_bytes(self) -> None: self.assert_fsspec_equal( log_record.attributes["gen_ai.input.messages_ref"], - '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"},{"type":"generic_bytes","bytes":"aGVsbG8="}]}]', + '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"},{"type":"generic_bytes","bytes":"aGVsbG8="}]}]\n', + ) + + def test_upload_json(self) -> None: + hook = FsspecUploadCompletionHook( + base_path=BASE_PATH, upload_format="json" + ) + self.addCleanup(hook.shutdown) + log_record = LogRecord() + + hook.on_completion( + inputs=FAKE_INPUTS, + outputs=FAKE_OUTPUTS, + system_instruction=FAKE_SYSTEM_INSTRUCTION, + log_record=log_record, + ) + hook.shutdown() + + ref_uri: str = log_record.attributes["gen_ai.input.messages_ref"] + self.assertTrue( + ref_uri.endswith(".json"), f"{ref_uri=} does not end with .json" + ) + + self.assert_fsspec_equal( + ref_uri, + '[{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"}]},{"role":"assistant","parts":[{"arguments":{"city":"Paris"},"name":"get_capital","id":"get_capital_0","type":"tool_call"}]},{"role":"user","parts":[{"response":{"capital":"Paris"},"id":"get_capital_0","type":"tool_call_response"}]}]\n', + ) + + def test_upload_jsonlines(self) -> None: + hook = FsspecUploadCompletionHook( + base_path=BASE_PATH, upload_format="jsonl" + ) + self.addCleanup(hook.shutdown) + log_record = LogRecord() + + hook.on_completion( + inputs=FAKE_INPUTS, + outputs=FAKE_OUTPUTS, + system_instruction=FAKE_SYSTEM_INSTRUCTION, + log_record=log_record, + ) + hook.shutdown() + + ref_uri: str = log_record.attributes["gen_ai.input.messages_ref"] + self.assertTrue( + ref_uri.endswith(".jsonl"), f"{ref_uri=} does not end with .jsonl" + ) + + self.assert_fsspec_equal( + ref_uri, + """\ +{"role":"user","parts":[{"content":"What is the capital of France?","type":"text"}],"index":0} +{"role":"assistant","parts":[{"arguments":{"city":"Paris"},"name":"get_capital","id":"get_capital_0","type":"tool_call"}],"index":1} +{"role":"user","parts":[{"response":{"capital":"Paris"},"id":"get_capital_0","type":"tool_call_response"}],"index":2} +""", ) diff --git a/uv.lock b/uv.lock index 7d46cd4836..e844f53b93 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.13'", @@ -11,6 +11,7 @@ resolution-markers = [ [manifest] members = [ + "opentelemetry-exporter-credential-provider-gcp", "opentelemetry-exporter-prometheus-remote-write", "opentelemetry-exporter-richconsole", "opentelemetry-instrumentation", @@ -2401,6 +2402,22 @@ dependencies = [ { name = "typing-extensions" }, ] +[[package]] +name = "opentelemetry-exporter-credential-provider-gcp" +source = { editable = "exporter/opentelemetry-exporter-credential-provider-gcp" } +dependencies = [ + { name = "google-auth" }, + { name = "grpcio" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "google-auth", specifier = ">=2.25" }, + { name = "grpcio", specifier = ">=1.67.0" }, + { name = "requests", specifier = ">=2.7" }, +] + [[package]] name = "opentelemetry-exporter-prometheus-remote-write" source = { editable = "exporter/opentelemetry-exporter-prometheus-remote-write" }