diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 7e1df8fef..4e3feb237 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -7,6 +7,12 @@ on: - "release/v*" - ci-workflow workflow_dispatch: # be able to run the workflow on demand + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string env: AWS_DEFAULT_REGION: us-east-1 STAGING_ECR_REGISTRY: 637423224110.dkr.ecr.us-east-1.amazonaws.com @@ -34,6 +40,8 @@ jobs: steps: - name: Checkout Repo @ SHA - ${{ github.sha }} uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + ref: ${{ inputs.ref || github.sha }} - name: Get Python Distro Output id: python_output diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml new file mode 100644 index 000000000..551ee8d42 --- /dev/null +++ b/.github/workflows/nightly-build.yml @@ -0,0 +1,121 @@ +name: Nightly Upstream Snapshot Build + +on: + schedule: + - cron: "21 3 * * *" + workflow_dispatch: + push: + branches: + - zhaez/nightly-build + +env: + BRANCH_NAME: nightly-dependency-updates + +jobs: + update-and-create-pr: + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.check_changes.outputs.has_changes }} + otel_python_version: ${{ steps.get_versions.outputs.otel_python_version }} + otel_contrib_version: ${{ steps.get_versions.outputs.otel_contrib_version }} + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 + with: + python-version: '3.11' + + - name: Install build tools + run: | + python -m pip install --upgrade pip + pip install toml requests packaging + + - name: Get latest upstream versions + id: get_versions + run: python scripts/get_upstream_versions.py + + - name: Check for breaking changes + id: breaking_changes + env: + OTEL_PYTHON_VERSION: ${{ steps.get_versions.outputs.otel_python_version }} + OTEL_CONTRIB_VERSION: ${{ steps.get_versions.outputs.otel_contrib_version }} + run: python scripts/find_breaking_changes.py + + - name: Configure git and create branch + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - name: Check out dependency update branch + run: | + if git ls-remote --exit-code --heads origin "$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME already exists, checking out..." + git checkout "$BRANCH_NAME" + else + echo "Branch $BRANCH_NAME does not exist, creating new branch..." + git checkout -b "$BRANCH_NAME" + fi + + - name: Update dependencies + env: + OTEL_PYTHON_VERSION: ${{ steps.get_versions.outputs.otel_python_version }} + OTEL_CONTRIB_VERSION: ${{ steps.get_versions.outputs.otel_contrib_version }} + run: python scripts/update_dependencies.py + + - name: Check for changes and commit + id: check_changes + run: | + if git diff --quiet; then + echo "No dependency updates needed" + echo "has_changes=false" >> $GITHUB_OUTPUT + else + echo "Dependencies were updated" + echo "has_changes=true" >> $GITHUB_OUTPUT + + git add aws-opentelemetry-distro/pyproject.toml + git commit -m "chore: update OpenTelemetry dependencies to ${{ steps.get_versions.outputs.otel_python_version }}/${{ steps.get_versions.outputs.otel_contrib_version }}" + git push origin "$BRANCH_NAME" + fi + + - name: Create or update PR + if: steps.check_changes.outputs.has_changes == 'true' + run: | + PR_BODY="Automated update of OpenTelemetry dependencies. + + **Updated versions:** + - OpenTelemetry Python: ${{ steps.get_versions.outputs.otel_python_version }} + - OpenTelemetry Contrib: ${{ steps.get_versions.outputs.otel_contrib_version }} + + **Upstream releases with breaking changes:** + ${{ steps.breaking_changes.outputs.breaking_changes_info }}" + + if gh pr view "$BRANCH_NAME" > /dev/null 2>&1; then + echo "PR already exists, updating description..." + gh pr edit "$BRANCH_NAME" --body "$PR_BODY" + else + echo "Creating new PR..." + gh pr create \ + --title "Nightly dependency update: OpenTelemetry ${{ steps.get_versions.outputs.otel_python_version }}/${{ steps.get_versions.outputs.otel_contrib_version }}" \ + --body "$PR_BODY" \ + --base main \ + --head "$BRANCH_NAME" + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-and-test: + needs: update-and-create-pr + if: needs.update-and-create-pr.outputs.has_changes == 'true' + uses: ./.github/workflows/main-build.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ref: nightly-dependency-updates \ No newline at end of file diff --git a/aws-opentelemetry-distro/pyproject.toml b/aws-opentelemetry-distro/pyproject.toml index 414b09221..6a30a3640 100644 --- a/aws-opentelemetry-distro/pyproject.toml +++ b/aws-opentelemetry-distro/pyproject.toml @@ -26,62 +26,62 @@ classifiers = [ ] dependencies = [ - "opentelemetry-api == 1.33.1", - "opentelemetry-sdk == 1.33.1", - "opentelemetry-exporter-otlp-proto-grpc == 1.33.1", - "opentelemetry-exporter-otlp-proto-http == 1.33.1", - "opentelemetry-propagator-b3 == 1.33.1", - "opentelemetry-propagator-jaeger == 1.33.1", - "opentelemetry-exporter-otlp-proto-common == 1.33.1", - "opentelemetry-sdk-extension-aws == 2.0.2", - "opentelemetry-propagator-aws-xray == 1.0.1", - "opentelemetry-distro == 0.54b1", - "opentelemetry-processor-baggage == 0.54b1", - "opentelemetry-propagator-ot-trace == 0.54b1", - "opentelemetry-instrumentation == 0.54b1", - "opentelemetry-instrumentation-aws-lambda == 0.54b1", - "opentelemetry-instrumentation-aio-pika == 0.54b1", - "opentelemetry-instrumentation-aiohttp-client == 0.54b1", - "opentelemetry-instrumentation-aiopg == 0.54b1", - "opentelemetry-instrumentation-asgi == 0.54b1", - "opentelemetry-instrumentation-asyncpg == 0.54b1", - "opentelemetry-instrumentation-boto == 0.54b1", - "opentelemetry-instrumentation-boto3sqs == 0.54b1", - "opentelemetry-instrumentation-botocore == 0.54b1", - "opentelemetry-instrumentation-celery == 0.54b1", - "opentelemetry-instrumentation-confluent-kafka == 0.54b1", - "opentelemetry-instrumentation-dbapi == 0.54b1", - "opentelemetry-instrumentation-django == 0.54b1", - "opentelemetry-instrumentation-elasticsearch == 0.54b1", - "opentelemetry-instrumentation-falcon == 0.54b1", - "opentelemetry-instrumentation-fastapi == 0.54b1", - "opentelemetry-instrumentation-flask == 0.54b1", - "opentelemetry-instrumentation-grpc == 0.54b1", - "opentelemetry-instrumentation-httpx == 0.54b1", - "opentelemetry-instrumentation-jinja2 == 0.54b1", - "opentelemetry-instrumentation-kafka-python == 0.54b1", - "opentelemetry-instrumentation-logging == 0.54b1", - "opentelemetry-instrumentation-mysql == 0.54b1", - "opentelemetry-instrumentation-mysqlclient == 0.54b1", - "opentelemetry-instrumentation-pika == 0.54b1", - "opentelemetry-instrumentation-psycopg2 == 0.54b1", - "opentelemetry-instrumentation-pymemcache == 0.54b1", - "opentelemetry-instrumentation-pymongo == 0.54b1", - "opentelemetry-instrumentation-pymysql == 0.54b1", - "opentelemetry-instrumentation-pyramid == 0.54b1", - "opentelemetry-instrumentation-redis == 0.54b1", - "opentelemetry-instrumentation-remoulade == 0.54b1", - "opentelemetry-instrumentation-requests == 0.54b1", - "opentelemetry-instrumentation-sqlalchemy == 0.54b1", - "opentelemetry-instrumentation-sqlite3 == 0.54b1", - "opentelemetry-instrumentation-starlette == 0.54b1", - "opentelemetry-instrumentation-system-metrics == 0.54b1", - "opentelemetry-instrumentation-tornado == 0.54b1", - "opentelemetry-instrumentation-tortoiseorm == 0.54b1", - "opentelemetry-instrumentation-urllib == 0.54b1", - "opentelemetry-instrumentation-urllib3 == 0.54b1", - "opentelemetry-instrumentation-wsgi == 0.54b1", - "opentelemetry-instrumentation-cassandra == 0.54b1", + "opentelemetry-api == 1.38.0", + "opentelemetry-sdk == 1.38.0", + "opentelemetry-exporter-otlp-proto-grpc == 1.38.0", + "opentelemetry-exporter-otlp-proto-http == 1.38.0", + "opentelemetry-propagator-b3 == 1.38.0", + "opentelemetry-propagator-jaeger == 1.38.0", + "opentelemetry-exporter-otlp-proto-common == 1.38.0", + "opentelemetry-sdk-extension-aws == 2.1.0", + "opentelemetry-propagator-aws-xray == 1.0.2", + "opentelemetry-distro == 0.59b0", + "opentelemetry-processor-baggage == 0.59b0", + "opentelemetry-propagator-ot-trace == 0.59b0", + "opentelemetry-instrumentation == 0.59b0", + "opentelemetry-instrumentation-aws-lambda == 0.59b0", + "opentelemetry-instrumentation-aio-pika == 0.59b0", + "opentelemetry-instrumentation-aiohttp-client == 0.59b0", + "opentelemetry-instrumentation-aiopg == 0.59b0", + "opentelemetry-instrumentation-asgi == 0.59b0", + "opentelemetry-instrumentation-asyncpg == 0.59b0", + "opentelemetry-instrumentation-boto == 0.59b0", + "opentelemetry-instrumentation-boto3sqs == 0.59b0", + "opentelemetry-instrumentation-botocore == 0.59b0", + "opentelemetry-instrumentation-celery == 0.59b0", + "opentelemetry-instrumentation-confluent-kafka == 0.59b0", + "opentelemetry-instrumentation-dbapi == 0.59b0", + "opentelemetry-instrumentation-django == 0.59b0", + "opentelemetry-instrumentation-elasticsearch == 0.59b0", + "opentelemetry-instrumentation-falcon == 0.59b0", + "opentelemetry-instrumentation-fastapi == 0.59b0", + "opentelemetry-instrumentation-flask == 0.59b0", + "opentelemetry-instrumentation-grpc == 0.59b0", + "opentelemetry-instrumentation-httpx == 0.59b0", + "opentelemetry-instrumentation-jinja2 == 0.59b0", + "opentelemetry-instrumentation-kafka-python == 0.59b0", + "opentelemetry-instrumentation-logging == 0.59b0", + "opentelemetry-instrumentation-mysql == 0.59b0", + "opentelemetry-instrumentation-mysqlclient == 0.59b0", + "opentelemetry-instrumentation-pika == 0.59b0", + "opentelemetry-instrumentation-psycopg2 == 0.59b0", + "opentelemetry-instrumentation-pymemcache == 0.59b0", + "opentelemetry-instrumentation-pymongo == 0.59b0", + "opentelemetry-instrumentation-pymysql == 0.59b0", + "opentelemetry-instrumentation-pyramid == 0.59b0", + "opentelemetry-instrumentation-redis == 0.59b0", + "opentelemetry-instrumentation-remoulade == 0.59b0", + "opentelemetry-instrumentation-requests == 0.59b0", + "opentelemetry-instrumentation-sqlalchemy == 0.59b0", + "opentelemetry-instrumentation-sqlite3 == 0.59b0", + "opentelemetry-instrumentation-starlette == 0.59b0", + "opentelemetry-instrumentation-system-metrics == 0.59b0", + "opentelemetry-instrumentation-tornado == 0.59b0", + "opentelemetry-instrumentation-tortoiseorm == 0.59b0", + "opentelemetry-instrumentation-urllib == 0.59b0", + "opentelemetry-instrumentation-urllib3 == 0.59b0", + "opentelemetry-instrumentation-wsgi == 0.59b0", + "opentelemetry-instrumentation-cassandra == 0.59b0", ] [project.optional-dependencies] diff --git a/scripts/find_breaking_changes.py b/scripts/find_breaking_changes.py new file mode 100644 index 000000000..ca0028264 --- /dev/null +++ b/scripts/find_breaking_changes.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import requests +import sys +import os +import re +from packaging import version + +def get_current_version_from_pyproject(): + """Extract current OpenTelemetry versions from pyproject.toml.""" + try: + with open('aws-opentelemetry-distro/pyproject.toml', 'r', encoding='utf-8') as file: + content = file.read() + + # Find first opentelemetry-api version (core version) + api_match = re.search(r'"opentelemetry-api == ([^"]*)"', content) + current_core_version = api_match.group(1) if api_match else None + + # Find first opentelemetry-distro version (contrib version) + distro_match = re.search(r'"opentelemetry-distro == ([^"]*)"', content) + current_contrib_version = distro_match.group(1) if distro_match else None + + return current_core_version, current_contrib_version + + except Exception as error: + print(f"Error reading current versions: {error}") + return None, None + +def get_releases_with_breaking_changes(repo, current_version, new_version): + """Get releases between current and new version that mention breaking changes.""" + try: + response = requests.get( + f'https://api.github.com/repos/open-telemetry/{repo}/releases', + timeout=30 + ) + response.raise_for_status() + + releases = response.json() + breaking_releases = [] + + for release in releases: + release_version = release['tag_name'].lstrip('v') + + # Check if this release is between current and new version + try: + if (version.parse(release_version) > version.parse(current_version) and + version.parse(release_version) <= version.parse(new_version)): + + # Check if release notes mention breaking changes + body = release.get('body', '').lower() + if any(keyword in body for keyword in ['breaking change', 'breaking changes', 'breaking:', 'breaking']): + breaking_releases.append({ + 'version': release_version, + 'name': release['name'], + 'url': release['html_url'], + 'body': release.get('body', '') + }) + except Exception: + # Skip releases with invalid version formats + continue + + return breaking_releases + + except requests.RequestException as request_error: + print(f"Warning: Could not get releases for {repo}: {request_error}") + return [] + +def main(): + new_core_version = os.environ.get('OTEL_PYTHON_VERSION') + new_contrib_version = os.environ.get('OTEL_CONTRIB_VERSION') + + if not new_core_version or not new_contrib_version: + print("Error: OTEL_PYTHON_VERSION and OTEL_CONTRIB_VERSION environment variables required") + sys.exit(1) + + current_core_version, current_contrib_version = get_current_version_from_pyproject() + + if not current_core_version or not current_contrib_version: + print("Could not determine current versions") + sys.exit(1) + + print(f"Checking for breaking changes:") + print(f"Core: {current_core_version} → {new_core_version}") + print(f"Contrib: {current_contrib_version} → {new_contrib_version}") + + # Check both repos for breaking changes + core_breaking = get_releases_with_breaking_changes('opentelemetry-python', current_core_version, new_core_version) + contrib_breaking = get_releases_with_breaking_changes('opentelemetry-python-contrib', current_contrib_version, new_contrib_version) + + # Output for GitHub Actions + breaking_info = "" + + if core_breaking: + breaking_info += "**opentelemetry-python:**\\n" + for release in core_breaking: + breaking_info += f"- [{release['name']}]({release['url']})\\n" + + if contrib_breaking: + breaking_info += "**opentelemetry-python-contrib:**\\n" + for release in contrib_breaking: + breaking_info += f"- [{release['name']}]({release['url']})\\n" + + # Set GitHub output + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output_file: + output_file.write(f'breaking_changes_info={breaking_info}\n') + +if __name__ == '__main__': + main() diff --git a/scripts/get_upstream_versions.py b/scripts/get_upstream_versions.py new file mode 100644 index 000000000..c0e95ed6c --- /dev/null +++ b/scripts/get_upstream_versions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +import sys + +import requests + + +def get_latest_otel_versions(): + """Get latest OpenTelemetry versions from GitHub releases.""" + try: + # Query GitHub API for latest release + response = requests.get( + "https://api.github.com/repos/open-telemetry/opentelemetry-python/releases/latest", timeout=30 + ) + response.raise_for_status() + + release_data = response.json() + release_title = release_data["name"] + + # Parse "Version 1.37.0/0.58b0" format + match = re.search(r"Version\s+(\d+\.\d+\.\d+)/(\d+\.\d+b\d+)", release_title) + if not match: + print(f"Could not parse release title: {release_title}") + sys.exit(1) + + otel_python_version = match.group(1) + otel_contrib_version = match.group(2) + + return otel_python_version, otel_contrib_version + + except requests.RequestException as request_error: + print(f"Error getting OpenTelemetry versions: {request_error}") + sys.exit(1) + + +def main(): + otel_python_version, otel_contrib_version = get_latest_otel_versions() + + print(f"OTEL_PYTHON_VERSION={otel_python_version}") + print(f"OTEL_CONTRIB_VERSION={otel_contrib_version}") + + # Write to GitHub output if in CI + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + output_file.write(f"otel_python_version={otel_python_version}\n") + output_file.write(f"otel_contrib_version={otel_contrib_version}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/update_dependencies.py b/scripts/update_dependencies.py new file mode 100644 index 000000000..7c2da71be --- /dev/null +++ b/scripts/update_dependencies.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +import sys + +import requests + +# Dependencies that use the first version number (opentelemetry-python) +PYTHON_CORE_DEPS = [ + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-propagator-b3", + "opentelemetry-propagator-jaeger", + "opentelemetry-exporter-otlp-proto-common", +] + +# Dependencies that use the second version number (opentelemetry-python-contrib) +CONTRIB_DEPS = [ + "opentelemetry-distro", + "opentelemetry-processor-baggage", + "opentelemetry-propagator-ot-trace", + "opentelemetry-instrumentation", + "opentelemetry-instrumentation-aws-lambda", + "opentelemetry-instrumentation-aio-pika", + "opentelemetry-instrumentation-aiohttp-client", + "opentelemetry-instrumentation-aiopg", + "opentelemetry-instrumentation-asgi", + "opentelemetry-instrumentation-asyncpg", + "opentelemetry-instrumentation-boto", + "opentelemetry-instrumentation-boto3sqs", + "opentelemetry-instrumentation-botocore", + "opentelemetry-instrumentation-celery", + "opentelemetry-instrumentation-confluent-kafka", + "opentelemetry-instrumentation-dbapi", + "opentelemetry-instrumentation-django", + "opentelemetry-instrumentation-elasticsearch", + "opentelemetry-instrumentation-falcon", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-instrumentation-flask", + "opentelemetry-instrumentation-grpc", + "opentelemetry-instrumentation-httpx", + "opentelemetry-instrumentation-jinja2", + "opentelemetry-instrumentation-kafka-python", + "opentelemetry-instrumentation-logging", + "opentelemetry-instrumentation-mysql", + "opentelemetry-instrumentation-mysqlclient", + "opentelemetry-instrumentation-pika", + "opentelemetry-instrumentation-psycopg2", + "opentelemetry-instrumentation-pymemcache", + "opentelemetry-instrumentation-pymongo", + "opentelemetry-instrumentation-pymysql", + "opentelemetry-instrumentation-pyramid", + "opentelemetry-instrumentation-redis", + "opentelemetry-instrumentation-remoulade", + "opentelemetry-instrumentation-requests", + "opentelemetry-instrumentation-sqlalchemy", + "opentelemetry-instrumentation-sqlite3", + "opentelemetry-instrumentation-starlette", + "opentelemetry-instrumentation-system-metrics", + "opentelemetry-instrumentation-tornado", + "opentelemetry-instrumentation-tortoiseorm", + "opentelemetry-instrumentation-urllib", + "opentelemetry-instrumentation-urllib3", + "opentelemetry-instrumentation-wsgi", + "opentelemetry-instrumentation-cassandra", +] + +# AWS-specific packages with independent versioning +AWS_DEPS = [ + "opentelemetry-sdk-extension-aws", + "opentelemetry-propagator-aws-xray", +] + + +def get_latest_version(package_name): + """Get the latest version of a package from PyPI.""" + try: + response = requests.get(f"https://pypi.org/pypi/{package_name}/json", timeout=30) + response.raise_for_status() + data = response.json() + return data["info"]["version"] + except requests.RequestException as request_error: + print(f"Warning: Could not get latest version for {package_name}: {request_error}") + return None + + +def main(): + otel_python_version = os.environ.get("OTEL_PYTHON_VERSION") + otel_contrib_version = os.environ.get("OTEL_CONTRIB_VERSION") + + if not otel_python_version or not otel_contrib_version: + print("Error: OTEL_PYTHON_VERSION and OTEL_CONTRIB_VERSION environment variables required") + sys.exit(1) + + pyproject_path = "aws-opentelemetry-distro/pyproject.toml" + + try: + with open(pyproject_path, "r", encoding="utf-8") as input_file: + content = input_file.read() + + updated = False + + # Update opentelemetry-python dependencies + for dep in PYTHON_CORE_DEPS: + pattern = rf'"{re.escape(dep)} == [^"]*"' + replacement = f'"{dep} == {otel_python_version}"' + if re.search(pattern, content): + content = re.sub(pattern, replacement, content) + updated = True + + # Update opentelemetry-python-contrib dependencies + for dep in CONTRIB_DEPS: + pattern = rf'"{re.escape(dep)} == [^"]*"' + replacement = f'"{dep} == {otel_contrib_version}"' + if re.search(pattern, content): + content = re.sub(pattern, replacement, content) + updated = True + + # Update dependencies with independent versioning + for dep in AWS_DEPS: + latest_version = get_latest_version(dep) + if latest_version: + pattern = rf'"{re.escape(dep)} == [^"]*"' + replacement = f'"{dep} == {latest_version}"' + if re.search(pattern, content): + content = re.sub(pattern, replacement, content) + updated = True + print(f"Updated {dep} to {latest_version}") + + if updated: + with open(pyproject_path, "w", encoding="utf-8") as output_file: + output_file.write(content) + print(f"Dependencies updated to Python {otel_python_version} / Contrib {otel_contrib_version}") + else: + print("No OpenTelemetry dependencies found to update") + + except (OSError, IOError) as file_error: + print(f"Error updating dependencies: {file_error}") + sys.exit(1) + + +if __name__ == "__main__": + main()