Skip to content

Commit 72683ea

Browse files
docs: update generated documentation (#3313)
Co-authored-by: liferoad <7833268+liferoad@users.noreply.github.com>
1 parent 63ebaa1 commit 72683ea

File tree

1 file changed

+8
-8
lines changed

1 file changed

+8
-8
lines changed

v2/datastream-to-bigquery/README_Cloud_Datastream_to_BigQuery.md

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,15 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat
3131

3232
### Required parameters
3333

34-
* **inputFilePattern**: The file location for Datastream file output in Cloud Storage, in the format `gs://<BUCKET_NAME>/<ROOT_PATH>/`.
3534
* **inputFileFormat**: The format of the output files produced by Datastream. Allowed values are `avro` and `json`. Defaults to `avro`.
36-
* **gcsPubSubSubscription**: The Pub/Sub subscription used by Cloud Storage to notify Dataflow of new files available for processing, in the format: `projects/<PROJECT_ID>/subscriptions/<SUBSCRIPTION_NAME>`.
3735
* **outputStagingDatasetTemplate**: The name of the dataset that contains staging tables. This parameter supports templates, for example `{_metadata_dataset}_log` or `my_dataset_log`. Normally, this parameter is a dataset name. Defaults to `{_metadata_dataset}`. Note: For MySQL sources, the database name is mapped to `{_metadata_schema}` instead of `{_metadata_dataset}`.
3836
* **outputDatasetTemplate**: The name of the dataset that contains the replica tables. This parameter supports templates, for example `{_metadata_dataset}` or `my_dataset`. Normally, this parameter is a dataset name. Defaults to `{_metadata_dataset}`. Note: For MySQL sources, the database name is mapped to `{_metadata_schema}` instead of `{_metadata_dataset}`.
3937
* **deadLetterQueueDirectory**: The path that Dataflow uses to write the dead-letter queue output. This path must not be in the same path as the Datastream file output. Defaults to `empty`.
4038

4139
### Optional parameters
4240

41+
* **inputFilePattern**: The file location for Datastream file output in Cloud Storage, in the format `gs://<BUCKET_NAME>/<ROOT_PATH>/`.
42+
* **gcsPubSubSubscription**: The Pub/Sub subscription used by Cloud Storage to notify Dataflow of new files available for processing, in the format: `projects/<PROJECT_ID>/subscriptions/<SUBSCRIPTION_NAME>`.
4343
* **streamName**: The name or the template for the stream to poll for schema information. Defaults to: {_metadata_stream}. The default value is usually enough.
4444
* **rfcStartDateTime**: The starting DateTime to use to fetch data from Cloud Storage (https://tools.ietf.org/html/rfc3339). Defaults to: `1970-01-01T00:00:00.00Z`.
4545
* **fileReadConcurrency**: The number of concurrent DataStream files to read. Default is `10`.
@@ -167,14 +167,14 @@ export REGION=us-central1
167167
export TEMPLATE_SPEC_GCSPATH="gs://$BUCKET_NAME/templates/flex/Cloud_Datastream_to_BigQuery"
168168

169169
### Required
170-
export INPUT_FILE_PATTERN=<inputFilePattern>
171170
export INPUT_FILE_FORMAT=avro
172-
export GCS_PUB_SUB_SUBSCRIPTION=<gcsPubSubSubscription>
173171
export OUTPUT_STAGING_DATASET_TEMPLATE={_metadata_dataset}
174172
export OUTPUT_DATASET_TEMPLATE={_metadata_dataset}
175173
export DEAD_LETTER_QUEUE_DIRECTORY=""
176174

177175
### Optional
176+
export INPUT_FILE_PATTERN=<inputFilePattern>
177+
export GCS_PUB_SUB_SUBSCRIPTION=<gcsPubSubSubscription>
178178
export STREAM_NAME=<streamName>
179179
export RFC_START_DATE_TIME=1970-01-01T00:00:00.00Z
180180
export FILE_READ_CONCURRENCY=10
@@ -254,14 +254,14 @@ export BUCKET_NAME=<bucket-name>
254254
export REGION=us-central1
255255

256256
### Required
257-
export INPUT_FILE_PATTERN=<inputFilePattern>
258257
export INPUT_FILE_FORMAT=avro
259-
export GCS_PUB_SUB_SUBSCRIPTION=<gcsPubSubSubscription>
260258
export OUTPUT_STAGING_DATASET_TEMPLATE={_metadata_dataset}
261259
export OUTPUT_DATASET_TEMPLATE={_metadata_dataset}
262260
export DEAD_LETTER_QUEUE_DIRECTORY=""
263261

264262
### Optional
263+
export INPUT_FILE_PATTERN=<inputFilePattern>
264+
export GCS_PUB_SUB_SUBSCRIPTION=<gcsPubSubSubscription>
265265
export STREAM_NAME=<streamName>
266266
export RFC_START_DATE_TIME=1970-01-01T00:00:00.00Z
267267
export FILE_READ_CONCURRENCY=10
@@ -340,12 +340,12 @@ resource "google_dataflow_flex_template_job" "cloud_datastream_to_bigquery" {
340340
name = "cloud-datastream-to-bigquery"
341341
region = var.region
342342
parameters = {
343-
inputFilePattern = "<inputFilePattern>"
344343
inputFileFormat = "avro"
345-
gcsPubSubSubscription = "<gcsPubSubSubscription>"
346344
outputStagingDatasetTemplate = "{_metadata_dataset}"
347345
outputDatasetTemplate = "{_metadata_dataset}"
348346
deadLetterQueueDirectory = ""
347+
# inputFilePattern = "<inputFilePattern>"
348+
# gcsPubSubSubscription = "<gcsPubSubSubscription>"
349349
# streamName = "<streamName>"
350350
# rfcStartDateTime = "1970-01-01T00:00:00.00Z"
351351
# fileReadConcurrency = "10"

0 commit comments

Comments
 (0)