Skip to content

Commit 5d6dcee

Browse files
Amar3ttoAmar3tto
authored
Add Load Tests CoGBK Flink Batch Go workflow (#28660)
* Add Load Tests CoGBK Flink Batch Go workflow * Change cron * Refactoring * Refactoring
1 parent b10546c commit 5d6dcee

File tree

4 files changed

+207
-0
lines changed

4 files changed

+207
-0
lines changed

‎.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: LoadTests Go CoGBK Flink Batch
17+
18+
on:
19+
issue_comment:
20+
types: [created]
21+
schedule:
22+
- cron: '10 14 * * *'
23+
workflow_dispatch:
24+
25+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: read
29+
checks: read
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: read
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}'
44+
cancel-in-progress: true
45+
46+
env:
47+
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
GCLOUD_ZONE: us-central1-a
51+
CLUSTER_NAME: beam-loadtests-go-cogbk-flink-batch-${{ github.run_id }}
52+
GCS_BUCKET: gs://beam-flink-cluster
53+
FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz
54+
HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
55+
FLINK_TASKMANAGER_SLOTS: 1
56+
DETACHED_MODE: true
57+
HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
58+
JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest
59+
ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-cogbk-flink-batch-${{ github.run_id }}
60+
61+
jobs:
62+
beam_LoadTests_Go_CoGBK_Flink_Batch:
63+
if: |
64+
github.event_name == 'workflow_dispatch' ||
65+
github.event_name == 'schedule' ||
66+
github.event.comment.body == 'Run Load Tests Go CoGBK Flink Batch'
67+
runs-on: [self-hosted, ubuntu-20.04, main]
68+
timeout-minutes: 720
69+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
70+
strategy:
71+
matrix:
72+
job_name: ["beam_LoadTests_Go_CoGBK_Flink_Batch"]
73+
job_phrase: ["Run Load Tests Go CoGBK Flink Batch"]
74+
steps:
75+
- uses: actions/checkout@v3
76+
- name: Setup repository
77+
uses: ./.github/actions/setup-action
78+
with:
79+
comment_phrase: ${{ matrix.job_phrase }}
80+
github_token: ${{ secrets.GITHUB_TOKEN }}
81+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
82+
- name: Prepare test arguments
83+
uses: ./.github/actions/test-arguments-action
84+
with:
85+
test-type: load
86+
test-language: go
87+
argument-file-paths: |
88+
${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt
89+
${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt
90+
${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt
91+
arguments: |
92+
--job_name=load-tests-go-flink-batch-cogbk-$(date '+%m%d%H%M%S' --utc)
93+
- name: Start Flink with parallelism 5
94+
env:
95+
FLINK_NUM_WORKERS: 5
96+
run: |
97+
cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create
98+
# The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
99+
- name: run CoGBK Flink Batch Go Load Test 1 (multiple keys)
100+
timeout-minutes: 240
101+
uses: ./.github/actions/gradle-command-self-hosted-action
102+
with:
103+
gradle-command: :sdks:go:test:load:run
104+
arguments: |
105+
-PloadTest.mainClass=cogbk \
106+
-Prunner=FlinkRunner \
107+
'-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_1 }}' \
108+
- name: run CoGBK Flink Batch Go Load Test 2 (reiterate 4 times 10KB values)
109+
timeout-minutes: 240
110+
uses: ./.github/actions/gradle-command-self-hosted-action
111+
with:
112+
gradle-command: :sdks:go:test:load:run
113+
arguments: |
114+
-PloadTest.mainClass=cogbk \
115+
-Prunner=FlinkRunner \
116+
'-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_2 }}' \
117+
- name: run CoGBK Flink Batch Go Load Test 3 (reiterate 4 times 2MB values)
118+
timeout-minutes: 240
119+
uses: ./.github/actions/gradle-command-self-hosted-action
120+
with:
121+
gradle-command: :sdks:go:test:load:run
122+
arguments: |
123+
-PloadTest.mainClass=cogbk \
124+
-Prunner=FlinkRunner \
125+
'-PloadTest.args=${{ env.beam_LoadTests_Go_CoGBK_Flink_Batch_test_arguments_3 }}'
126+
- name: Teardown Flink
127+
if: always()
128+
run: |
129+
${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete

‎.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_MultipleKey.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--influx_measurement=go_batch_cogbk_1
18+
--influx_namespace=flink
19+
--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}''
20+
--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
21+
--iterations=1
22+
--parallelism=5
23+
--endpoint=localhost:8099
24+
--environment_type=DOCKER
25+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
26+
--runner=FlinkRunner

‎.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_10KB.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--influx_measurement=go_batch_cogbk_2
18+
--influx_namespace=flink
19+
--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":200000,\"hot_key_fraction\":1}''
20+
--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
21+
--iterations=4
22+
--parallelism=5
23+
--endpoint=localhost:8099
24+
--environment_type=DOCKER
25+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
26+
--runner=FlinkRunner

‎.github/workflows/load-tests-job-configs/go_CoGBK_Flink_Batch_Reiteration_2MB.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--influx_measurement=go_batch_cogbk_3
18+
--influx_namespace=flink
19+
--input_options=''{\"num_records\":20000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
20+
--co_input_options=''{\"num_records\":2000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
21+
--iterations=4
22+
--parallelism=5
23+
--endpoint=localhost:8099
24+
--environment_type=DOCKER
25+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
26+
--runner=FlinkRunner

0 commit comments

Comments
 (0)