diff --git a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json index a975cd1cd104..541dc4ea8e87 100644 --- a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json +++ b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "revision": 1 + "revision": 2 } diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml index 8ec3c2bc7aaf..1f3d51475664 100644 --- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml +++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml @@ -54,21 +54,22 @@ jobs: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') - runs-on: [self-hosted, ubuntu-20.04, main] + runs-on: ubuntu-latest timeout-minutes: 100 name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: job_name: ["beam_PostCommit_Yaml_Xlang_Direct"] job_phrase: ["Run Yaml_Xlang_Direct PostCommit"] + test_set: ["data", "databases", "messaging"] steps: - uses: actions/checkout@v4 - name: Setup repository uses: ./.github/actions/setup-action with: - comment_phrase: ${{ matrix.job_phrase }} + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_set }} github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_job: ${{ matrix.job_name }} ${{ matrix.test_set }} (${{ matrix.job_phrase }} ${{ matrix.test_set }}) - name: Setup environment uses: ./.github/actions/setup-environment-action with: @@ -79,7 +80,7 @@ jobs: - name: run PostCommit Yaml Xlang Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:postCommitYamlIntegrationTests -PbeamPythonExtra=ml_test + gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=ml_test - name: Archive Python Test Results uses: actions/upload-artifact@v4 if: failure() @@ -93,4 +94,5 @@ jobs: commit: '${{ env.prsha || env.GITHUB_SHA }}' comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} files: '**/pytest*.xml' - large_files: true \ No newline at end of file + large_files: true + diff --git a/sdks/python/apache_beam/yaml/extended_tests/enrichment.yaml b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/enrichment.yaml rename to sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/tfrecord.yaml b/sdks/python/apache_beam/yaml/extended_tests/data/tfrecord.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/tfrecord.yaml rename to sdks/python/apache_beam/yaml/extended_tests/data/tfrecord.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/bigquery.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/bigquery.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/jdbc.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/jdbc.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/jdbc.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/jdbc.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/mysql.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/mysql.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/mysql.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/mysql.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/oracle.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/oracle.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/oracle.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/oracle.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/postgres.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/postgres.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/postgres.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/postgres.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/spanner.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/spanner.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/spanner.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/spanner.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/sqlserver.yaml b/sdks/python/apache_beam/yaml/extended_tests/databases/sqlserver.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/sqlserver.yaml rename to sdks/python/apache_beam/yaml/extended_tests/databases/sqlserver.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/kafka.yaml b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/kafka.yaml rename to sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml diff --git a/sdks/python/apache_beam/yaml/extended_tests/pubsub.yaml b/sdks/python/apache_beam/yaml/extended_tests/messaging/pubsub.yaml similarity index 100% rename from sdks/python/apache_beam/yaml/extended_tests/pubsub.yaml rename to sdks/python/apache_beam/yaml/extended_tests/messaging/pubsub.yaml diff --git a/sdks/python/apache_beam/yaml/integration_tests.py b/sdks/python/apache_beam/yaml/integration_tests.py index f6c60ae5f121..6824105737a8 100644 --- a/sdks/python/apache_beam/yaml/integration_tests.py +++ b/sdks/python/apache_beam/yaml/integration_tests.py @@ -228,7 +228,7 @@ def temp_mysql_database(): with the MySQL database during setup. Exception: Any other exception encountered during the setup process. """ - with MySqlContainer() as mysql_container: + with MySqlContainer(init=True) as mysql_container: try: # Make connection to temp database and create tmp table engine = sqlalchemy.create_engine(mysql_container.get_connection_url()) @@ -440,12 +440,12 @@ def temp_kafka_server(): Exception: If there's an error starting the Kafka container or interacting with the temporary Kafka server. """ - try: - with KafkaContainer() as kafka_container: + with KafkaContainer() as kafka_container: + try: yield kafka_container.get_bootstrap_server() - except Exception as err: - logging.error("Error interacting with temporary Kakfa Server: %s", err) - raise err + except Exception as err: + logging.error("Error interacting with temporary Kakfa Server: %s", err) + raise err @contextlib.contextmanager diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index accacb1953bb..c8f02262d3fd 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -138,13 +138,13 @@ tasks.register("yamlIntegrationTests") { doLast { exec { executable 'sh' - args '-c', "${envdir}/bin/pytest -v apache_beam/yaml/integration_tests.py --deselect apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0" + args '-c', "${envdir}/bin/pytest -v apache_beam/yaml/integration_tests.py" } } } tasks.register("postCommitYamlIntegrationTests") { - description "Runs postcommit integration tests for yaml pipelines." + description "Runs postcommit integration tests for yaml pipelines - parameterized by yamlTestSet." dependsOn installGcpTest // Need to build all expansion services referenced in apache_beam/yaml/*.* @@ -155,9 +155,28 @@ tasks.register("postCommitYamlIntegrationTests") { dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build" doLast { - exec { - executable 'sh' - args '-c', "${envdir}/bin/pytest -v apache_beam/yaml/integration_tests.py --deselect apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0 --test_files_dir='extended_tests'" + def testSetInput = project.findProperty('yamlTestSet') ?: 'data,databases,messaging' + def testSetsToRun = testSetInput.tokenize(',').collect { it.trim() }.findAll { !it.isEmpty() } + testSetsToRun.each { currentTestSet -> + def test_files_dir + + switch (currentTestSet) { + case 'data': + test_files_dir = 'extended_tests/data' + break + case 'databases': + test_files_dir = 'extended_tests/databases' + break + case 'messaging': + test_files_dir = 'extended_tests/messaging' + break + default: + throw StopExecutionException("Unknown yamlTestSet: ${testSet}. Must be one of 'data', 'databases', or 'messaging'.") + } + exec { + executable 'sh' + args '-c', "${envdir}/bin/pytest -v apache_beam/yaml/integration_tests.py --deselect apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0 --test_files_dir='${test_files_dir}'" + } } } } diff --git a/sdks/python/setup.py b/sdks/python/setup.py index bfcbd4b6b8af..d8072f417013 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -436,7 +436,8 @@ def get_portability_package_data(): 'virtualenv-clone>=0.5,<1.0', 'mysql-connector-python>=9.3.0', 'python-tds>=1.16.1', - 'sqlalchemy-pytds>=1.0.2' + 'sqlalchemy-pytds>=1.0.2', + 'oracledb>=3.1.1' ], 'gcp': [ 'cachetools>=3.1.0,<6',