Skip to content

Commit 1c74621

Browse files
mohamedawnallahmohamedawnallah
authored
workflows: add ubuntu-latest to beam_PreCommit_Python_ML, beam_PostCommit_Python, and beam_PreCommit_Python_Coverage (#35816)
* workflows: add `ubuntu-latest` to postcommit/ml * workflows: include os runner as part of job name and test results * sdks/python: properly make milvus as extra dependency * sdks/python: fix linting issues * sdks/python: fix formatting issues * .github: trigger postcommit python * workflows: properly show job names * .github: trigger beam postcommit python * workflows: add `ubuntu-latest` to `beam_PreCommit_Python_Coverage` * actions+sdks/python: upgrade pip * workflows: exclude `ubuntu-latest` from `beam_PostCommit_Python` * sdks/python: force tests to run only on precommit * Revert "actions+sdks/python: upgrade pip" This reverts commit c8d88b0. * workflows: run python 3.12 only on precommit ml `ubuntu-latest` os * test-suites/direct/common.gradle: remove milvus postcommit gradle hook
1 parent 55bbbb3 commit 1c74621

File tree

5 files changed

+31
-59
lines changed

5 files changed

+31
-59
lines changed

‎.github/workflows/beam_PostCommit_Python.yml‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -53,21 +53,15 @@ env:
5353

5454
jobs:
5555
beam_PostCommit_Python:
56-
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
56+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
5757
runs-on: ${{ matrix.os }}
5858
timeout-minutes: 240
5959
strategy:
6060
fail-fast: false
6161
matrix:
62-
job_name: [beam_PostCommit_Python]
63-
job_phrase: [Run Python PostCommit]
62+
job_name: ['beam_PostCommit_Python']
63+
job_phrase: ['Run Python PostCommit']
6464
python_version: ['3.9', '3.10', '3.11', '3.12']
65-
# Run on both self-hosted and GitHub-hosted runners.
66-
# Some tests (marked require_docker_in_docker) can't run on Beam's
67-
# self-hosted runners due to Docker-in-Docker environment constraint.
68-
# These tests will only execute on ubuntu-latest (GitHub-hosted).
69-
# Context: https://github.com/apache/beam/pull/35585
70-
# Temporary removed the ubuntu-latest env till resolving deps issues.
7165
os: [[self-hosted, ubuntu-20.04, highmem22]]
7266
if: |
7367
github.event_name == 'workflow_dispatch' ||
@@ -81,7 +75,7 @@ jobs:
8175
with:
8276
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
8377
github_token: ${{ secrets.GITHUB_TOKEN }}
84-
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
78+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
8579
- name: Setup environment
8680
uses: ./.github/actions/setup-environment-action
8781
with:
@@ -106,19 +100,15 @@ jobs:
106100
arguments: |
107101
-Pjava21Home=$JAVA_HOME_21_X64 \
108102
-PuseWheelDistribution \
109-
-Pposargs="${{
110-
contains(matrix.os, 'self-hosted') &&
111-
'-m (not require_docker_in_docker)' ||
112-
'-m require_docker_in_docker'
113-
}}" \
103+
-Pposargs="-m (not require_docker_in_docker)" \
114104
-PpythonVersion=${{ matrix.python_version }} \
115105
env:
116106
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
117107
- name: Archive Python Test Results
118108
uses: actions/upload-artifact@v4
119109
if: failure()
120110
with:
121-
name: Python ${{ matrix.python_version }} Test Results
111+
name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})
122112
path: '**/pytest*.xml'
123113
- name: Publish Python Test Results
124114
uses: EnricoMi/publish-unit-test-result-action@v2
@@ -128,3 +118,4 @@ jobs:
128118
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
129119
files: '**/pytest*.xml'
130120
large_files: true
121+
check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

‎.github/workflows/beam_PreCommit_Python_Coverage.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,7 @@ jobs:
7171
# self-hosted runners due to Docker-in-Docker environment constraint.
7272
# These tests will only execute on ubuntu-latest (GitHub-hosted).
7373
# Context: https://github.com/apache/beam/pull/35585
74-
# Temporary removed the ubuntu-latest env till resolving deps issues.
75-
os: [[self-hosted, ubuntu-20.04, highmem]]
74+
os: [[self-hosted, ubuntu-20.04, highmem], [ubuntu-latest]]
7675
timeout-minutes: 180
7776
if: |
7877
github.event_name == 'push' ||

‎.github/workflows/beam_PreCommit_Python_ML.yml‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ env:
5757

5858
jobs:
5959
beam_PreCommit_Python_ML:
60-
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
60+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
6161
runs-on: ${{ matrix.os }}
6262
timeout-minutes: 180
6363
strategy:
@@ -70,9 +70,18 @@ jobs:
7070
# Some tests (marked require_docker_in_docker) can't run on Beam's
7171
# self-hosted runners due to Docker-in-Docker environment constraint.
7272
# These tests will only execute on ubuntu-latest (GitHub-hosted).
73-
# Context: https://github.com/apache/beam/pull/35585
74-
# Temporary removed the ubuntu-latest env till resolving deps issues.
75-
os: [[self-hosted, ubuntu-20.04, main]]
73+
# Context: https://github.com/apache/beam/pull/35585.
74+
os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
75+
exclude:
76+
# Temporary exclude Python 3.9, 3.10, 3.11 from ubuntu-latest. This
77+
# results in pip dependency resolution exceeded maximum depth issue.
78+
# Context: https://github.com/apache/beam/pull/35816.
79+
- python_version: '3.9'
80+
os: [ubuntu-latest]
81+
- python_version: '3.10'
82+
os: [ubuntu-latest]
83+
- python_version: '3.11'
84+
os: [ubuntu-latest]
7685
if: |
7786
github.event_name == 'push' ||
7887
github.event_name == 'pull_request_target' ||
@@ -86,7 +95,7 @@ jobs:
8695
with:
8796
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
8897
github_token: ${{ secrets.GITHUB_TOKEN }}
89-
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
98+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
9099
- name: Setup environment
91100
uses: ./.github/actions/setup-environment-action
92101
with:
@@ -113,7 +122,7 @@ jobs:
113122
uses: actions/upload-artifact@v4
114123
if: failure()
115124
with:
116-
name: Python ${{ matrix.python_version }} Test Results
125+
name: Python ${{ matrix.python_version }} Test Results ${{ matrix.os }}
117126
path: '**/pytest*.xml'
118127
- name: Publish Python Test Results
119128
uses: EnricoMi/publish-unit-test-result-action@v2
@@ -123,3 +132,4 @@ jobs:
123132
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
124133
files: '**/pytest*.xml'
125134
large_files: true
135+
check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

‎sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@ def test_empty_input_chunks(self):
574574

575575
expected_chunks = []
576576

577-
with TestPipeline(is_integration_test=True) as p:
577+
with TestPipeline() as p:
578578
result = (p | beam.Create(test_chunks) | Enrichment(handler))
579579
assert_that(
580580
result,
@@ -702,7 +702,7 @@ def test_filtered_search_with_cosine_similarity_and_batching(self):
702702
embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
703703
]
704704

705-
with TestPipeline(is_integration_test=True) as p:
705+
with TestPipeline() as p:
706706
result = (p | beam.Create(test_chunks) | Enrichment(handler))
707707
assert_that(
708708
result,
@@ -807,7 +807,7 @@ def test_filtered_search_with_bm25_full_text_and_batching(self):
807807
embedding=Embedding())
808808
]
809809

810-
with TestPipeline(is_integration_test=True) as p:
810+
with TestPipeline() as p:
811811
result = (p | beam.Create(test_chunks) | Enrichment(handler))
812812
assert_that(
813813
result,
@@ -948,7 +948,7 @@ def test_vector_search_with_euclidean_distance(self):
948948
embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
949949
]
950950

951-
with TestPipeline(is_integration_test=True) as p:
951+
with TestPipeline() as p:
952952
result = (p | beam.Create(test_chunks) | Enrichment(handler))
953953
assert_that(
954954
result,
@@ -1088,7 +1088,7 @@ def test_vector_search_with_inner_product_similarity(self):
10881088
embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
10891089
]
10901090

1091-
with TestPipeline(is_integration_test=True) as p:
1091+
with TestPipeline() as p:
10921092
result = (p | beam.Create(test_chunks) | Enrichment(handler))
10931093
assert_that(
10941094
result,
@@ -1153,7 +1153,7 @@ def test_keyword_search_with_inner_product_sparse_embedding(self):
11531153
sparse_embedding=([1, 2, 3, 4], [0.05, 0.41, 0.05, 0.41])))
11541154
]
11551155

1156-
with TestPipeline(is_integration_test=True) as p:
1156+
with TestPipeline() as p:
11571157
result = (p | beam.Create(test_chunks) | Enrichment(handler))
11581158
assert_that(
11591159
result,
@@ -1226,7 +1226,7 @@ def test_hybrid_search(self):
12261226
embedding=Embedding(dense_embedding=[0.1, 0.2, 0.3]))
12271227
]
12281228

1229-
with TestPipeline(is_integration_test=True) as p:
1229+
with TestPipeline() as p:
12301230
result = (p | beam.Create(test_chunks) | Enrichment(handler))
12311231
assert_that(
12321232
result,

‎sdks/python/test-suites/direct/common.gradle‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -419,33 +419,6 @@ task feastIntegrationTest {
419419
}
420420
}
421421

422-
// Integration tests that require Docker-in-Docker capabilities.
423-
// These tests are marked with the `require_docker_in_docker` pytest marker
424-
// because they rely on Docker-in-Docker configurations that are not supported
425-
// on Beam's self-hosted GitHub Actions runners. Docker-in-Docker works on
426-
// ubuntu-latest GitHub-hosted runners but not on self-hosted environments due
427-
// to containerization architecture differences.
428-
// Context: https://github.com/apache/beam/pull/35585
429-
task dockerInDockerIntegrationTest {
430-
dependsOn 'installGcpTest'
431-
dependsOn ':sdks:python:sdist'
432-
433-
doLast {
434-
def testOpts = basicTestOpts
435-
def argMap = [
436-
"test_opts": testOpts,
437-
"suite": "postCommitIT-direct-py${pythonVersionSuffix}",
438-
"collect": "require_docker_in_docker",
439-
"runner": "TestDirectRunner",
440-
]
441-
def cmdArgs = mapToArgString(argMap)
442-
exec {
443-
executable 'sh'
444-
args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
445-
}
446-
}
447-
}
448-
449422
// Add all the RunInference framework IT tests to this gradle task that runs on Direct Runner Post commit suite.
450423
project.tasks.register("inferencePostCommitIT") {
451424
dependsOn = [
@@ -456,7 +429,6 @@ project.tasks.register("inferencePostCommitIT") {
456429
'transformersInferenceTest',
457430
'testcontainersTest',
458431
'feastIntegrationTest',
459-
'dockerInDockerIntegrationTest',
460432
// (TODO) https://github.com/apache/beam/issues/25799
461433
// uncomment tfx bsl tests once tfx supports protobuf 4.x
462434
// 'tfxInferenceTest',

0 commit comments

Comments
 (0)