Skip to content

Commit ffd7d02

Browse files
liferoadliferoad
authored
added run_rc_validation_python_mobile_gaming.yml (#34472)
1 parent 3ac053f commit ffd7d02

File tree

1 file changed

+342
-0
lines changed

1 file changed

+342
-0
lines changed

‎.github/workflows/run_rc_validation_python_mobile_gaming.yml‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,342 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Run Python Mobile Gaming RC Validation
19+
20+
on:
21+
workflow_dispatch:
22+
inputs:
23+
RELEASE_VER:
24+
description: 'Beam Release Version (e.g., 2.64.0)'
25+
required: true
26+
default: '2.64.0'
27+
RC_NUM:
28+
description: 'Release Candidate number (e.g., 1)'
29+
required: true
30+
default: '1'
31+
APACHE_CONTENTS_REPO:
32+
description: 'Apache Staging Repository URL for Java Injector (e.g., https://repository.apache.org/content/repositories/orgapachebeam-1234)'
33+
required: true
34+
35+
# This allows a subsequently queued workflow run to interrupt previous runs
36+
concurrency:
37+
group: '${{ github.workflow }} @ ${{ github.event.inputs.RELEASE_VER }}-${{ github.event.inputs.RC_NUM }}'
38+
cancel-in-progress: true
39+
40+
# Setting explicit permissions for the action
41+
permissions:
42+
actions: write
43+
pull-requests: write # Needed for setup-action potentially
44+
checks: write
45+
contents: read # Needs read to checkout the code
46+
deployments: read
47+
id-token: write # Required for GCP Workload Identity Federation
48+
issues: write
49+
discussions: read
50+
packages: read
51+
pages: read
52+
repository-projects: read
53+
security-events: read
54+
statuses: read
55+
56+
env:
57+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
58+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
59+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
60+
# Define unique names for resources based on run ID to avoid collisions
61+
RUN_ID_SUFFIX: ${{ github.run_id }}_${{ github.run_attempt }}
62+
BQ_DATASET: mobilegaming_py_rc_${{ github.run_id }}_${{ github.run_attempt }}
63+
PUBSUB_TOPIC: mobilegaming_py_rc_${{ github.run_id }}_${{ github.run_attempt }}
64+
# Set GCP Project ID, Bucket, Region as constants
65+
GCP_PROJECT_ID: 'apache-beam-testing'
66+
GCS_BUCKET: 'gs://rc-validation-migration-tests' # Includes gs:// prefix
67+
GCE_REGION: 'us-central1'
68+
# Java Injector specific envs
69+
APACHE_REPO_URL: ${{ github.event.inputs.APACHE_CONTENTS_REPO }}
70+
# Release specific envs
71+
RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VER }}
72+
RC_NUM: ${{ github.event.inputs.RC_NUM }}
73+
RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}"
74+
# Python specific envs
75+
PYTHON_VERSION: '3.9' # Specify desired Python version
76+
BEAM_PYTHON_SDK_TAR_GZ: apache_beam-${{ github.event.inputs.RELEASE_VER }}.tar.gz
77+
BEAM_SOURCE_TAR_GZ: apache_beam-${{ github.event.inputs.RELEASE_VER }}-source-release.zip
78+
APACHE_DIST_URL_BASE: https://dist.apache.org/repos/dist/dev/beam/${{ github.event.inputs.RELEASE_VER }}
79+
# Default duration for GameStats fixed window
80+
GAME_STATS_WINDOW_DURATION: 60
81+
82+
jobs:
83+
run_python_mobile_gaming_rc_validation:
84+
name: Run Python Mobile Gaming RC Validation (${{ github.event.inputs.RELEASE_VER }} RC${{ github.event.inputs.RC_NUM }})
85+
runs-on: [self-hosted, ubuntu-20.04, main] # Assuming same runner type needed
86+
timeout-minutes: 180 # Increased timeout for Python steps + Java injector
87+
steps:
88+
- name: Checkout code at RC tag
89+
uses: actions/checkout@v4
90+
with:
91+
ref: ${{ env.RC_TAG }}
92+
93+
# Standard setup actions (consider if setup-action is needed or if manual setup is sufficient)
94+
- name: Setup environment
95+
uses: ./.github/actions/setup-environment-action
96+
with:
97+
java-version: 11
98+
99+
- name: Setup Python
100+
uses: actions/setup-python@v5
101+
with:
102+
python-version: ${{ env.PYTHON_VERSION }}
103+
104+
- name: Install Dependencies
105+
run: |
106+
sudo apt-get update
107+
sudo apt-get install -y wget unzip sha256sum
108+
109+
- name: Set up Cloud SDK
110+
uses: google-github-actions/setup-gcloud@v2
111+
112+
- name: Download RC Artifacts
113+
run: |
114+
echo "Downloading from ${{ env.APACHE_DIST_URL_BASE }}"
115+
wget ${{ env.APACHE_DIST_URL_BASE }}/python/${{ env.BEAM_PYTHON_SDK_TAR_GZ }}
116+
wget ${{ env.APACHE_DIST_URL_BASE }}/python/${{ env.BEAM_PYTHON_SDK_TAR_GZ }}.sha512
117+
wget ${{ env.APACHE_DIST_URL_BASE }}/${{ env.BEAM_SOURCE_TAR_GZ }}
118+
wget ${{ env.APACHE_DIST_URL_BASE }}/${{ env.BEAM_SOURCE_TAR_GZ }}.sha512
119+
shell: bash
120+
121+
- name: Verify Hashes
122+
run: |
123+
echo "Verifying sha512 checksums..."
124+
sha512sum -c ${{ env.BEAM_PYTHON_SDK_TAR_GZ }}.sha512
125+
sha512sum -c ${{ env.BEAM_SOURCE_TAR_GZ }}.sha512
126+
shell: bash
127+
128+
- name: Build Python SDK from Source
129+
run: |
130+
echo "Building Python SDK sdist..."
131+
tar -xzf ${{ env.BEAM_SOURCE_TAR_GZ }}
132+
# Navigate into the extracted directory (assuming it matches the tarball name without .tar.gz)
133+
SOURCE_DIR=$(basename ${{ env.BEAM_SOURCE_TAR_GZ }} .tar.gz)
134+
cd ${SOURCE_DIR}/sdks/python
135+
python setup.py sdist
136+
# Move the built dist back to the working directory
137+
mv dist/apache-beam-${{ env.RELEASE_VERSION }}.tar.gz ../../../
138+
shell: bash
139+
140+
- name: Setup Python Virtual Environment
141+
run: |
142+
echo "Setting up Python virtual environment..."
143+
python -m venv beam_env
144+
source beam_env/bin/activate
145+
pip install --upgrade pip setuptools wheel
146+
echo "Virtual environment ready."
147+
shell: bash
148+
149+
- name: Install Python SDK
150+
run: |
151+
echo "Installing built Python SDK: apache-beam-${{ env.RELEASE_VERSION }}.tar.gz"
152+
source beam_env/bin/activate
153+
# Install base SDK
154+
pip install apache-beam-${{ env.RELEASE_VERSION }}.tar.gz
155+
# Install with GCP extras
156+
pip install apache-beam-${{ env.RELEASE_VERSION }}.tar.gz[gcp]
157+
echo "SDK installed."
158+
pip freeze # Log installed packages
159+
shell: bash
160+
161+
# ================== GCP Resource Setup ==================
162+
- name: Create BigQuery Dataset
163+
run: |
164+
echo "Creating BigQuery dataset: ${{ env.BQ_DATASET }} in project ${{ env.GCP_PROJECT_ID }}"
165+
bq mk --project_id=${{ env.GCP_PROJECT_ID }} ${{ env.BQ_DATASET }}
166+
shell: bash
167+
168+
- name: Create GCS Bucket (if needed - reusing input bucket)
169+
run: |
170+
echo "Ensuring GCS Bucket exists: ${{ env.GCS_BUCKET }} in project ${{ env.GCP_PROJECT_ID }}"
171+
# gsutil mb command creates if it doesn't exist, fails gracefully if it does
172+
gsutil mb -p ${{ env.GCP_PROJECT_ID }} ${{ env.GCS_BUCKET }} || echo "Bucket ${{ env.GCS_BUCKET }} likely already exists."
173+
shell: bash
174+
175+
- name: Create PubSub Topic
176+
run: |
177+
echo "Creating PubSub topic: ${{ env.PUBSUB_TOPIC }} in project ${{ env.GCP_PROJECT_ID }}"
178+
gcloud pubsub topics create --project=${{ env.GCP_PROJECT_ID }} ${{ env.PUBSUB_TOPIC }}
179+
shell: bash
180+
181+
# ================== Java Data Injector ==================
182+
- name: Configure Maven Settings for Injector
183+
run: |
184+
mkdir -p ~/.m2
185+
cat <<EOF > ~/.m2/settings.xml
186+
<settings>
187+
<profiles>
188+
<profile>
189+
<id>release-repo</id>
190+
<activation>
191+
<activeByDefault>true</activeByDefault>
192+
</activation>
193+
<repositories>
194+
<repository>
195+
<id>Release ${{ env.RELEASE_VERSION }} RC${{ env.RC_NUM }}</id>
196+
<name>Release ${{ env.RELEASE_VERSION }} RC${{ env.RC_NUM }}</name>
197+
<url>${{ env.APACHE_REPO_URL }}</url>
198+
</repository>
199+
</repositories>
200+
</profile>
201+
</profiles>
202+
</settings>
203+
EOF
204+
echo "Maven settings.xml configured for Java Injector."
205+
shell: bash
206+
207+
- name: Run Java Injector
208+
run: |
209+
echo "Running Java Injector..."
210+
# Generate a dummy project to get dependencies - adjust archetype version if needed
211+
mvn archetype:generate \
212+
-DarchetypeGroupId=org.apache.beam \
213+
-DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
214+
-DarchetypeVersion=${{ env.RELEASE_VERSION }} \
215+
-DgroupId=org.example \
216+
-DartifactId=injector-temp \
217+
-Dversion="0.1" \
218+
-Dpackage=org.apache.beam.examples \
219+
-DinteractiveMode=false \
220+
-DarchetypeCatalog=internal
221+
222+
cd injector-temp
223+
# Compile and run the injector
224+
mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
225+
-Dexec.args="${{ env.GCP_PROJECT_ID }} ${{ env.PUBSUB_TOPIC }} none" \
226+
-Dmaven.wagon.http.retryHandler.count=3 \
227+
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
228+
229+
cd .. # Return to the root directory
230+
echo "Java Injector finished."
231+
shell: bash
232+
233+
# ================== Leaderboard Tests ==================
234+
- name: Run Leaderboard (Direct Runner)
235+
run: |
236+
echo "Running Leaderboard with DirectRunner..."
237+
source beam_env/bin/activate
238+
python -m apache_beam.examples.complete.game.leader_board \
239+
--project=${{ env.GCP_PROJECT_ID }} \
240+
--topic projects/${{ env.GCP_PROJECT_ID }}/topics/${{ env.PUBSUB_TOPIC }} \
241+
--dataset ${{ env.BQ_DATASET }}
242+
shell: bash
243+
244+
- name: Inspect Leaderboard Results (Direct Runner)
245+
run: |
246+
echo "Inspecting BigQuery results for Leaderboard (DirectRunner)..."
247+
sleep 60 # Allow some time for data to propagate in BQ
248+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.leader_board_users
249+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.leader_board_teams
250+
shell: bash
251+
252+
- name: Run Leaderboard (Dataflow Runner)
253+
run: |
254+
echo "Running Leaderboard with DataflowRunner..."
255+
source beam_env/bin/activate
256+
GCS_BUCKET_NAME_NO_PREFIX=$(echo ${{ env.GCS_BUCKET }} | sed 's/^gs:\/\///')
257+
python -m apache_beam.examples.complete.game.leader_board \
258+
--project=${{ env.GCP_PROJECT_ID }} \
259+
--region=${{ env.GCE_REGION }} \
260+
--topic projects/${{ env.GCP_PROJECT_ID }}/topics/${{ env.PUBSUB_TOPIC }} \
261+
--dataset ${{ env.BQ_DATASET }} \
262+
--runner DataflowRunner \
263+
--temp_location=${{ env.GCS_BUCKET }}/temp/ \
264+
--sdk_location=apache-beam-${{ env.RELEASE_VERSION }}.tar.gz
265+
shell: bash
266+
267+
- name: Inspect Leaderboard Results (Dataflow Runner)
268+
run: |
269+
echo "Inspecting BigQuery results for Leaderboard (DataflowRunner)..."
270+
# Dataflow jobs might take longer to write results
271+
sleep 180 # Allow more time for Dataflow job and BQ writes
272+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.leader_board_users
273+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.leader_board_teams
274+
shell: bash
275+
276+
# ================== GameStats Tests ==================
277+
- name: Run GameStats (Direct Runner)
278+
run: |
279+
echo "Running GameStats with DirectRunner..."
280+
source beam_env/bin/activate
281+
python -m apache_beam.examples.complete.game.game_stats \
282+
--project=${{ env.GCP_PROJECT_ID }} \
283+
--topic projects/${{ env.GCP_PROJECT_ID }}/topics/${{ env.PUBSUB_TOPIC }} \
284+
--dataset ${{ env.BQ_DATASET }} \
285+
--fixed_window_duration ${{ env.GAME_STATS_WINDOW_DURATION }}
286+
shell: bash
287+
288+
- name: Inspect GameStats Results (Direct Runner)
289+
run: |
290+
echo "Inspecting BigQuery results for GameStats (DirectRunner)..."
291+
sleep 60 # Allow some time for data to propagate in BQ
292+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.game_stats_teams
293+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.game_stats_sessions
294+
shell: bash
295+
296+
- name: Run GameStats (Dataflow Runner)
297+
run: |
298+
echo "Running GameStats with DataflowRunner..."
299+
source beam_env/bin/activate
300+
GCS_BUCKET_NAME_NO_PREFIX=$(echo ${{ env.GCS_BUCKET }} | sed 's/^gs:\/\///')
301+
python -m apache_beam.examples.complete.game.game_stats \
302+
--project=${{ env.GCP_PROJECT_ID }} \
303+
--region=${{ env.GCE_REGION }} \
304+
--topic projects/${{ env.GCP_PROJECT_ID }}/topics/${{ env.PUBSUB_TOPIC }} \
305+
--dataset ${{ env.BQ_DATASET }} \
306+
--runner DataflowRunner \
307+
--temp_location=${{ env.GCS_BUCKET }}/temp/ \
308+
--sdk_location=apache-beam-${{ env.RELEASE_VERSION }}.tar.gz \
309+
--fixed_window_duration ${{ env.GAME_STATS_WINDOW_DURATION }}
310+
shell: bash
311+
312+
- name: Inspect GameStats Results (Dataflow Runner)
313+
run: |
314+
echo "Inspecting BigQuery results for GameStats (DataflowRunner)..."
315+
# Dataflow jobs might take longer to write results
316+
sleep 180 # Allow more time for Dataflow job and BQ writes
317+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.game_stats_teams
318+
bq head -n 10 ${{ env.GCP_PROJECT_ID }}:${{ env.BQ_DATASET }}.game_stats_sessions
319+
shell: bash
320+
321+
# ================== Cleanup ==================
322+
- name: Cleanup BigQuery Dataset
323+
if: always()
324+
run: |
325+
echo "Deleting BigQuery dataset: ${{ env.BQ_DATASET }} in project ${{ env.GCP_PROJECT_ID }}"
326+
bq rm --project_id=${{ env.GCP_PROJECT_ID }} -f -r ${{ env.BQ_DATASET }} || echo "Failed to delete BQ dataset ${{ env.BQ_DATASET }}, continuing..."
327+
shell: bash
328+
329+
- name: Cleanup GCS Bucket Objects (Optional - depends on policy)
330+
if: always()
331+
run: |
332+
echo "Deleting objects in GCS Bucket: ${{ env.GCS_BUCKET }}/temp/"
333+
# Be cautious with bucket deletion; only delete temp contents if appropriate
334+
gsutil -m rm -r "${{ env.GCS_BUCKET }}/temp/" || echo "Failed to delete objects in GCS bucket temp folder, continuing..."
335+
shell: bash
336+
337+
- name: Cleanup PubSub Topic
338+
if: always()
339+
run: |
340+
echo "Deleting PubSub topic: ${{ env.PUBSUB_TOPIC }} in project ${{ env.GCP_PROJECT_ID }}"
341+
gcloud pubsub topics delete --project=${{ env.GCP_PROJECT_ID }} ${{ env.PUBSUB_TOPIC }} --quiet || echo "Failed to delete PubSub topic ${{ env.PUBSUB_TOPIC }}, continuing..."
342+
shell: bash

0 commit comments

Comments
 (0)