Skip to content

Commit c786cd0

Browse files
authored
[yaml]: uncomment WriteToKafka portion of test (#35277)
* uncomment kafka test and see if it runs with self hosted runner * try again but with streaming mode enabled * comment out the read portion
1 parent 5d13819 commit c786cd0

File tree

2 files changed

+24
-20
lines changed

2 files changed

+24
-20
lines changed
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run",
3-
"revision": 3
3+
"revision": 4
44
}

sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
# limitations under the License.
1616
#
1717

18-
# fixtures:
19-
# - name: TEMP_BOOTSTAP_SERVER
20-
# type: "apache_beam.yaml.integration_tests.temp_kafka_server"
18+
fixtures:
19+
- name: TEMP_BOOTSTAP_SERVER
20+
type: "apache_beam.yaml.integration_tests.temp_kafka_server"
2121

2222
pipelines:
2323
# Kafka write pipeline
@@ -30,23 +30,24 @@ pipelines:
3030
- {value: 123}
3131
- {value: 456}
3232
- {value: 789}
33-
# TODO(#35272) - fix kafka test
34-
# - type: MapToFields
35-
# config:
36-
# language: python
37-
# fields:
38-
# value:
39-
# callable: |
40-
# lambda row: str(row.value).encode('utf-8')
41-
# output_type: bytes
42-
# - type: WriteToKafka
43-
# config:
44-
# format: "RAW"
45-
# topic: "silly_topic"
46-
# bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}"
47-
# producer_config_updates:
48-
# linger.ms: "0"
33+
- type: MapToFields
34+
config:
35+
language: python
36+
fields:
37+
value:
38+
callable: |
39+
lambda row: str(row.value).encode('utf-8')
40+
output_type: bytes
41+
- type: WriteToKafka
42+
config:
43+
format: "RAW"
44+
topic: "silly_topic"
45+
bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}"
46+
producer_config_updates:
47+
linger.ms: "0"
4948

49+
# TODO(#35272): Figure out why this times out when running on github.
50+
# Locally runs fine.
5051
# Kafka read pipeline
5152
# Need a separate read pipeline to make sure the write pipeline is flushed
5253
# - pipeline:
@@ -77,5 +78,8 @@ pipelines:
7778
# - {value: "456"}
7879
# - {value: "789"}
7980

81+
options:
82+
streaming: true
83+
8084
# TODO: Error handling hard to trigger upon initial investigations. Need to
8185
# investigate more.

0 commit comments

Comments
 (0)