Skip to content

Commit 7e985e7

Browse files
committed
polished the test
1 parent ce1e8fd commit 7e985e7

File tree

2 files changed

+33
-13
lines changed

2 files changed

+33
-13
lines changed

sdks/python/apache_beam/transforms/sql.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ class SqlTransform(ExternalTransform):
7070
`apache_beam.examples.wordcount_xlang_sql`, `apache_beam.examples.sql_taxi`,
7171
and `apache_beam.transforms.sql_test`.
7272
73+
Full list of options for `sql_transform_schema` can be found in `configurationSchema` at
74+
https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.html.
75+
7376
For more details about Beam SQL in general, see the `Java transform
7477
<https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/extensions/sql/SqlTransform.html>`_,
7578
and the `documentation
@@ -81,7 +84,8 @@ def __init__(
8184
self,
8285
query: typing.Optional[str] = None,
8386
dialect: typing.Optional[str] = None,
84-
expansion_service: typing.Optional[typing.Union[str, BeamJarExpansionService]] = None,
87+
expansion_service: typing.Optional[typing.Union[
88+
str, BeamJarExpansionService]] = None,
8589
sql_transform_schema: typing.Optional[typing.NamedTuple] = None,
8690
):
8791
"""Creates a SqlTransform which will be expanded to Java's SqlTransform.

sdks/python/apache_beam/transforms/sql_test.py

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -210,20 +210,36 @@ def test_map(self):
210210
assert_that(out, equal_to([('alice', {'apples': 2, 'bananas': 3})]))
211211

212212
def test_sql_transform_schema_parameter(self):
213-
# Define the schema using the actual SqlTransformSchema
214-
from apache_beam.transforms.sql import SqlTransformSchema
215-
schema = SqlTransformSchema(
216-
query="""SELECT
217-
CAST(2 AS INT) AS `id`,
218-
CAST('bar' AS VARCHAR) AS `str`,
219-
CAST(1.618 AS DOUBLE) AS `flt`""",
220-
dialect=None,
221-
ddl=None)
222-
213+
# Test DDL
223214
with TestPipeline() as p:
224-
out = p | SqlTransform(sql_transform_schema=schema)
215+
input_data = [
216+
beam.Row(id=1, name='Alice'),
217+
beam.Row(id=2, name='Bob'),
218+
beam.Row(id=3, name='Charlie')
219+
]
220+
ddl_statement_pcoll = """
221+
CREATE EXTERNAL TABLE PCOLLECTION (
222+
id INT,
223+
name VARCHAR
224+
)
225+
TYPE 'beam'
226+
LOCATION 'pcollection'
227+
"""
228+
query_statement_pcoll = "SELECT id, name FROM PCOLLECTION WHERE id > 1"
229+
230+
schema = typing.NamedTuple(
231+
'SqlTransformSchema', [('query', query_statement_pcoll),
232+
('ddl', ddl_statement_pcoll)])
233+
234+
out = (
235+
p | beam.Create(input_data)
236+
| SqlTransform(sql_transform_schema=schema))
237+
225238
# Verify the output matches the query defined in the schema
226-
assert_that(out, equal_to([(2, "bar", 1.618)]))
239+
assert_that(
240+
out,
241+
equal_to([beam.Row(id=2, name='Bob'), beam.Row(id=3,
242+
name='Charlie')]))
227243

228244
def test_sql_transform_schema_parameter_with_warning(self):
229245
from apache_beam.transforms.sql import SqlTransformSchema

0 commit comments

Comments
 (0)