Skip to content

Commit 82ce53a

Browse files
liferoadliferoad
committed
Merge branch 'master' into grpc-options
2 parents 16d93fa + eba04b2 commit 82ce53a

File tree

161 files changed

+9849
-4046
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

161 files changed

+9849
-4046
lines changed

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ github:
5151

5252
protected_branches:
5353
master: {}
54+
release-2.69.0-postrelease: {}
5455
release-2.69: {}
5556
release-2.68.0-postrelease: {}
5657
release-2.68: {}

‎.github/REVIEWERS.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ labels:
4141
- chamikaramj
4242
- m-trieu
4343
- kennknowles
44-
- robertwb
4544
exclusionList: []
4645
- name: spanner
4746
reviewers:
@@ -76,6 +75,5 @@ fallbackReviewers:
7675
- jrmccluskey
7776
- kennknowles
7877
- liferoad
79-
- robertwb
8078
- shunping
8179
- tvalentyn

‎.github/trigger_files/IO_Iceberg_Integration_Tests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 2
3+
"modification": 3
44
}

‎.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run",
3-
"modification": 12
3+
"modification": 13
44
}

‎.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml‎

Lines changed: 75 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,20 +83,85 @@ jobs:
8383
- name: Install Kafka
8484
id: install_kafka
8585
run: |
86-
kubectl apply -k ${{ github.workspace }}/.test-infra/kafka/strimzi/02-kafka-persistent/overlays/gke-internal-load-balanced
87-
kubectl wait kafka beam-testing-cluster --for=condition=Ready --timeout=1800s
86+
echo "Deploying Kafka cluster using existing .test-infra/kubernetes/kafka-cluster configuration..."
87+
kubectl apply -R -f ${{ github.workspace }}/.test-infra/kubernetes/kafka-cluster/
88+
89+
# Wait for pods to be created and ready
90+
echo "Waiting for Kafka cluster to be ready..."
91+
sleep 180
92+
93+
# Check pod status
94+
echo "Checking pod status..."
95+
kubectl get pods -l app=kafka
96+
kubectl get pods -l app=zookeeper
97+
98+
# Wait for at least one Kafka pod to be ready
99+
echo "Waiting for Kafka pods to be ready..."
100+
kubectl wait --for=condition=ready pod -l app=kafka --timeout=300s || echo "Kafka pods not ready, continuing anyway"
101+
102+
# Wait for Zookeeper to be ready
103+
echo "Waiting for Zookeeper pods to be ready..."
104+
kubectl wait --for=condition=ready pod -l app=zookeeper --timeout=300s || echo "Zookeeper pods not ready, continuing anyway"
105+
88106
- name: Set up Kafka brokers
89107
id: set_brokers
90108
run: |
109+
echo "Setting up Kafka brokers for existing cluster configuration..."
91110
declare -a kafka_service_brokers
92111
declare -a kafka_service_brokers_ports
112+
93113
for INDEX in {0..2}; do
94-
kubectl wait svc/beam-testing-cluster-kafka-${INDEX} --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=1200s
95-
kafka_service_brokers[$INDEX]=$(kubectl get svc beam-testing-cluster-kafka-${INDEX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
96-
kafka_service_brokers_ports[$INDEX]=$(kubectl get svc beam-testing-cluster-kafka-${INDEX} -o jsonpath='{.spec.ports[0].port}')
114+
echo "Setting up broker ${INDEX}..."
115+
116+
# Try to get LoadBalancer IP
117+
LB_IP=$(kubectl get svc outside-${INDEX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
118+
119+
if [ -n "$LB_IP" ] && [ "$LB_IP" != "null" ]; then
120+
echo "Using LoadBalancer IP: $LB_IP"
121+
kafka_service_brokers[$INDEX]=$LB_IP
122+
else
123+
echo "LoadBalancer IP not available, using NodePort approach..."
124+
# Get the first node's internal IP
125+
NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
126+
kafka_service_brokers[$INDEX]=$NODE_IP
127+
fi
128+
129+
# Get the port
130+
PORT=$(kubectl get svc outside-${INDEX} -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || echo "9094")
131+
kafka_service_brokers_ports[$INDEX]=$PORT
132+
97133
echo "KAFKA_SERVICE_BROKER_${INDEX}=${kafka_service_brokers[$INDEX]}" >> $GITHUB_OUTPUT
98134
echo "KAFKA_SERVICE_BROKER_PORTS_${INDEX}=${kafka_service_brokers_ports[$INDEX]}" >> $GITHUB_OUTPUT
135+
136+
echo "Broker ${INDEX}: ${kafka_service_brokers[$INDEX]}:${kafka_service_brokers_ports[$INDEX]}"
99137
done
138+
139+
- name: Create Kafka topic
140+
id: create_topic
141+
run: |
142+
echo "Creating Kafka topic 'beam'..."
143+
144+
# Get the first available Kafka pod
145+
KAFKA_POD=$(kubectl get pods -l app=kafka -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
146+
147+
if [ -z "$KAFKA_POD" ]; then
148+
echo "No Kafka pods found, skipping topic creation"
149+
exit 0
150+
fi
151+
152+
echo "Using Kafka pod: $KAFKA_POD"
153+
154+
# Wait a bit more for the pod to be fully operational
155+
echo "Waiting for pod to be fully operational..."
156+
sleep 60
157+
158+
# Create the topic using the correct container and path
159+
echo "Creating topic 'beam'..."
160+
kubectl exec $KAFKA_POD -c broker -- /opt/kafka/bin/kafka-topics.sh --create --topic beam --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 || echo "Topic may already exist"
161+
162+
# Verify topic was created
163+
echo "Verifying topic creation..."
164+
kubectl exec $KAFKA_POD -c broker -- /opt/kafka/bin/kafka-topics.sh --list --zookeeper zookeeper:2181 || echo "Could not list topics"
100165
- name: Prepare test arguments
101166
uses: ./.github/actions/test-arguments-action
102167
with:
@@ -105,8 +170,11 @@ jobs:
105170
argument-file-paths: |
106171
${{ github.workspace }}/.github/workflows/performance-tests-pipeline-options/xlang_KafkaIO_Python.txt
107172
arguments: |
108-
--filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/
173+
--test_class=KafkaIOPerfTest
174+
--kafka_topic=beam
109175
--bootstrap_servers=${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_0 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_0 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_1 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_1 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_2 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_2 }}
176+
--read_timeout=3000
177+
--filename_prefix=gs://temp-storage-for-perf-tests/${{ matrix.job_name }}/${{github.run_id}}/
110178
- name: run shadowJar
111179
uses: ./.github/actions/gradle-command-self-hosted-action
112180
with:
@@ -120,4 +188,4 @@ jobs:
120188
-Prunner=DataflowRunner \
121189
-PloadTest.mainClass=apache_beam.io.external.xlang_kafkaio_perf_test \
122190
-PpythonVersion=3.9 \
123-
'-PloadTest.args=${{ env.beam_PerfTests_xlang_KafkaIO_Python_test_arguments_1 }}'
191+
'-PloadTest.args=${{ env.beam_PerfTests_xlang_KafkaIO_Python_test_arguments_1 }}'

‎.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,21 @@ jobs:
7979
with:
8080
java-version: default
8181
python-version: ${{ matrix.python_version }}
82+
- name: Authenticate to GCP
83+
uses: google-github-actions/auth@v3
84+
with:
85+
service_account: ${{ secrets.GCP_SA_EMAIL }}
86+
credentials_json: ${{ secrets.GCP_SA_KEY }}
87+
- name: Set up Cloud SDK
88+
uses: google-github-actions/setup-gcloud@v3
89+
- name: Configure Docker auth for GCR
90+
run: |
91+
gcloud --quiet auth configure-docker us.gcr.io
92+
gcloud --quiet auth configure-docker gcr.io
93+
gcloud auth list
94+
- name: Docker login to GCR (explicit)
95+
run: |
96+
gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us.gcr.io
8297
- name: Set PY_VER_CLEAN
8398
id: set_py_ver_clean
8499
run: |
@@ -105,7 +120,7 @@ jobs:
105120
uses: actions/upload-artifact@v4
106121
if: failure()
107122
with:
108-
name: Python Test Results
123+
name: Python Test Results ${{ matrix.python_version }}
109124
path: '**/pytest*.xml'
110125
- name: Publish Python Test Results
111126
uses: EnricoMi/publish-unit-test-result-action@v2

‎.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,11 @@
1616
# TODO(https://github.com/apache/beam/issues/32492): re-enable the suite
1717
# on cron and add release/trigger_all_tests.json to trigger path once fixed.
1818

19-
name: PostCommit XVR GoUsingJava Dataflow
19+
name: PostCommit XVR GoUsingJava Dataflow (disabled)
2020

2121
on:
22-
schedule:
23-
- cron: '45 5/6 * * *'
2422
pull_request_target:
25-
paths: ['.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json', 'release/trigger_all_tests.json']
23+
paths: ['.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json']
2624
workflow_dispatch:
2725

2826
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

‎.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ jobs:
8181
- "java:expansion-service:container:docker"
8282
steps:
8383
- uses: actions/checkout@v4
84+
- name: Free Disk Space (Ubuntu)
85+
uses: jlumbroso/[email protected]
8486
- name: Setup repository
8587
uses: ./.github/actions/setup-action
8688
with:

‎.github/workflows/beam_StressTests_Java_KafkaIO.yml‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,21 +80,86 @@ jobs:
8080
- name: Install Kafka
8181
id: install_kafka
8282
run: |
83-
kubectl apply -k ${{ github.workspace }}/.test-infra/kafka/strimzi/02-kafka-persistent/overlays/gke-internal-load-balanced
84-
kubectl wait kafka beam-testing-cluster --for=condition=Ready --timeout=1800s
83+
echo "Deploying Kafka cluster using existing .test-infra/kubernetes/kafka-cluster configuration..."
84+
kubectl apply -R -f ${{ github.workspace }}/.test-infra/kubernetes/kafka-cluster/
85+
86+
# Wait for pods to be created and ready
87+
echo "Waiting for Kafka cluster to be ready..."
88+
sleep 180
89+
90+
# Check pod status
91+
echo "Checking pod status..."
92+
kubectl get pods -l app=kafka
93+
kubectl get pods -l app=zookeeper
94+
95+
# Wait for at least one Kafka pod to be ready
96+
echo "Waiting for Kafka pods to be ready..."
97+
kubectl wait --for=condition=ready pod -l app=kafka --timeout=300s || echo "Kafka pods not ready, continuing anyway"
98+
99+
# Wait for Zookeeper to be ready
100+
echo "Waiting for Zookeeper pods to be ready..."
101+
kubectl wait --for=condition=ready pod -l app=zookeeper --timeout=300s || echo "Zookeeper pods not ready, continuing anyway"
102+
85103
- name: Set up Kafka brokers
86104
id: set_brokers
87105
run: |
106+
echo "Setting up Kafka brokers for existing cluster configuration..."
88107
declare -a kafka_service_brokers
89108
declare -a kafka_service_brokers_ports
109+
90110
for INDEX in {0..2}; do
91-
kubectl wait svc/beam-testing-cluster-kafka-${INDEX} --for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=1200s
92-
kafka_service_brokers[$INDEX]=$(kubectl get svc beam-testing-cluster-kafka-${INDEX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
93-
kafka_service_brokers_ports[$INDEX]=$(kubectl get svc beam-testing-cluster-kafka-${INDEX} -o jsonpath='{.spec.ports[0].port}')
111+
echo "Setting up broker ${INDEX}..."
112+
113+
# Try to get LoadBalancer IP
114+
LB_IP=$(kubectl get svc outside-${INDEX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
115+
116+
if [ -n "$LB_IP" ] && [ "$LB_IP" != "null" ]; then
117+
echo "Using LoadBalancer IP: $LB_IP"
118+
kafka_service_brokers[$INDEX]=$LB_IP
119+
else
120+
echo "LoadBalancer IP not available, using NodePort approach..."
121+
# Get the first node's internal IP
122+
NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
123+
kafka_service_brokers[$INDEX]=$NODE_IP
124+
fi
125+
126+
# Get the port
127+
PORT=$(kubectl get svc outside-${INDEX} -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || echo "9094")
128+
kafka_service_brokers_ports[$INDEX]=$PORT
129+
94130
echo "KAFKA_SERVICE_BROKER_${INDEX}=${kafka_service_brokers[$INDEX]}" >> $GITHUB_OUTPUT
95131
echo "KAFKA_SERVICE_BROKER_PORTS_${INDEX}=${kafka_service_brokers_ports[$INDEX]}" >> $GITHUB_OUTPUT
132+
133+
echo "Broker ${INDEX}: ${kafka_service_brokers[$INDEX]}:${kafka_service_brokers_ports[$INDEX]}"
96134
done
135+
136+
- name: Create Kafka topic
137+
id: create_topic
138+
run: |
139+
echo "Creating Kafka topic 'beam'..."
140+
141+
# Get the first available Kafka pod
142+
KAFKA_POD=$(kubectl get pods -l app=kafka -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
143+
144+
if [ -z "$KAFKA_POD" ]; then
145+
echo "No Kafka pods found, skipping topic creation"
146+
exit 0
147+
fi
148+
149+
echo "Using Kafka pod: $KAFKA_POD"
150+
151+
# Wait a bit more for the pod to be fully operational
152+
echo "Waiting for pod to be fully operational..."
153+
sleep 60
154+
155+
# Create the topic using the correct container and path
156+
echo "Creating topic 'beam'..."
157+
kubectl exec $KAFKA_POD -c broker -- /opt/kafka/bin/kafka-topics.sh --create --topic beam --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 || echo "Topic may already exist"
158+
159+
# Verify topic was created
160+
echo "Verifying topic creation..."
161+
kubectl exec $KAFKA_POD -c broker -- /opt/kafka/bin/kafka-topics.sh --list --zookeeper zookeeper:2181 || echo "Could not list topics"
97162
- name: run Kafka StressTest Large
98163
uses: ./.github/actions/gradle-command-self-hosted-action
99164
with:
100-
gradle-command: :it:kafka:KafkaStressTestLarge --info -DbootstrapServers="${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_0 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_0 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_1 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_1 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_2 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_2 }}" -DinfluxHost="http://10.128.0.96:8086" -DinfluxDatabase="beam_test_metrics" -DinfluxMeasurement="java_stress_test_kafka"
165+
gradle-command: :it:kafka:KafkaStressTestLarge --info -DbootstrapServers="${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_0 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_0 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_1 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_1 }},${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_2 }}:${{ steps.set_brokers.outputs.KAFKA_SERVICE_BROKER_PORTS_2 }}" -DinfluxHost="http://10.128.0.96:8086" -DinfluxDatabase="beam_test_metrics" -DinfluxMeasurement="java_stress_test_kafka"

‎.github/workflows/build_release_candidate.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ jobs:
332332
with:
333333
python-version: '3.9'
334334
- name: Install node
335-
uses: actions/setup-node@v5
335+
uses: actions/setup-node@v6
336336
with:
337337
node-version: '16'
338338
- name: Install Java 21

0 commit comments

Comments
 (0)