Skip to content

Commit df7ca4c

Browse files
rakeshcusatrakeshcusat
authored
Merge branch 'apache:master' into master
2 parents f7bc0e4 + d427d37 commit df7ca4c

File tree

309 files changed

+5443
-3223
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

309 files changed

+5443
-3223
lines changed

‎.github/trigger_files/IO_Iceberg_Integration_Tests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 2
3+
"modification": 3
44
}

‎.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 1
3+
"modification": 2
44
}

‎.github/trigger_files/beam_PostCommit_Java_Nexmark_Dataflow.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"modification": 1
2+
"modification": 2
33
}

‎.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,33 @@ jobs:
8484
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt
8585
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt
8686
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt
87+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Streaming_DistilBert_Base_Uncased.txt
88+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Batch_DistilBert_Base_Uncased.txt
8789
# The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
8890
- name: get current time
8991
run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
92+
- name: run Pytorch Sentiment Streaming using Hugging Face distilbert-base-uncased model
93+
uses: ./.github/actions/gradle-command-self-hosted-action
94+
timeout-minutes: 180
95+
with:
96+
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
97+
arguments: |
98+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \
99+
-Prunner=DataflowRunner \
100+
-PpythonVersion=3.10 \
101+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
102+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_6 }} --mode=streaming --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-streaming-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \
103+
- name: run Pytorch Sentiment Batch using Hugging Face distilbert-base-uncased model
104+
uses: ./.github/actions/gradle-command-self-hosted-action
105+
timeout-minutes: 180
106+
with:
107+
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
108+
arguments: |
109+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \
110+
-Prunner=DataflowRunner \
111+
-PpythonVersion=3.10 \
112+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
113+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_7 }} --mode=batch --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-batch-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \
90114
- name: run Pytorch Vision Classification with Resnet 101
91115
uses: ./.github/actions/gradle-command-self-hosted-action
92116
timeout-minutes: 180

‎.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,17 @@ jobs:
102102
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
103103
- name: Setup environment
104104
uses: ./.github/actions/setup-environment-action
105+
with:
106+
java-version: |
107+
8
108+
11
105109
- name: run PostCommit Java Nexmark Dataflow V2 (streaming = ${{ matrix.streaming }}) script
106110
uses: ./.github/actions/gradle-command-self-hosted-action
107111
with:
108112
gradle-command: :sdks:java:testing:nexmark:run
109113
arguments: |
114+
-PtestJavaVersion=8 \
115+
-Pjava8Home=$JAVA_HOME_8_X64 \
116+
-Pnexmark.runner.version=V2 \
110117
-Pnexmark.runner=:runners:google-cloud-dataflow-java \
111118
'${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}'

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Batch_DistilBert_Base_Uncased.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--region=us-central1
18+
--machine_type=n1-standard-2
19+
--num_workers=20
20+
--max_num_workers=250
21+
--disk_size_gb=50
22+
--autoscaling_algorithm=THROUGHPUT_BASED
23+
--staging_location=gs://temp-storage-for-perf-tests/loadtests
24+
--temp_location=gs://temp-storage-for-perf-tests/loadtests
25+
--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt
26+
--publish_to_big_query=true
27+
--metrics_dataset=beam_run_inference
28+
--metrics_table=torch_sentiment_distilbert_base_uncased_batch
29+
--input_options={}
30+
--influx_measurement=torch_sentiment_distilbert_base_uncased
31+
--device=CPU
32+
--input_file=gs://apache-beam-ml/testing/inputs/sentences_50k.txt
33+
--runner=DataflowRunner
34+
--model_path=distilbert-base-uncased-finetuned-sst-2-english
35+
--model_state_dict_path=gs://apache-beam-ml/models/huggingface.sentiment.distilbert-base-uncased.pth

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Streaming_DistilBert_Base_Uncased.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--region=us-central1
18+
--machine_type=n1-standard-2
19+
--num_workers=20
20+
--max_num_workers=250
21+
--disk_size_gb=50
22+
--autoscaling_algorithm=THROUGHPUT_BASED
23+
--staging_location=gs://temp-storage-for-perf-tests/loadtests
24+
--temp_location=gs://temp-storage-for-perf-tests/loadtests
25+
--requirements_file=apache_beam/ml/inference/torch_tests_requirements.txt
26+
--publish_to_big_query=true
27+
--metrics_dataset=beam_run_inference
28+
--metrics_table=torch_sentiment_distilbert_base_uncased
29+
--input_options={}
30+
--influx_measurement=torch_sentiment_distilbert_base_uncased
31+
--device=CPU
32+
--input_file=gs://apache-beam-ml/testing/inputs/sentences_50k.txt
33+
--runner=DataflowRunner
34+
--model_path=distilbert-base-uncased-finetuned-sst-2-english
35+
--model_state_dict_path=gs://apache-beam-ml/models/huggingface.sentiment.distilbert-base-uncased.pth

‎.github/workflows/tour_of_beam_backend_integration.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ on:
3434
- '.github/workflows/tour_of_beam_backend_integration.yml'
3535
- 'learning/tour-of-beam/backend/**'
3636
- 'playground/backend/**'
37+
schedule:
38+
- cron: '0 10 * * 6'
3739

3840
# This allows a subsequently queued workflow run to interrupt previous runs
3941
concurrency:

‎.pre-commit-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
# limitations under the License.
1212

1313
repos:
14-
- repo: https://github.com/pre-commit/mirrors-yapf
14+
- repo: https://github.com/google/yapf
1515
# this rev is a release tag in the repo above and corresponds with a yapf
1616
# version. make sure this matches the version of yapf in tox.ini.
17-
rev: v0.29.0
17+
rev: v0.43.0
1818
hooks:
1919
- id: yapf
2020
files: ^sdks/python/apache_beam/

‎.test-infra/tools/refresh_looker_metrics.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
("78", ["243", "244", "245", "246", "247"]), # PyTorch Resnet 101
4141
("79", ["248", "249", "250", "251", "252"]), # PyTorch Resnet 152
4242
("80", ["253", "254", "255", "256", "257"]), # PyTorch Resnet 152 Tesla T4
43+
("82", ["263", "264", "265", "266", "267"]), # PyTorch Sentiment Streaming DistilBERT base uncased
44+
("85", ["268", "269", "270", "271", "272"]), # PyTorch Sentiment Batch DistilBERT base uncased
4345
]
4446

4547

0 commit comments

Comments
 (0)