diff --git a/website/www/site/content/en/documentation/io/managed-io.md b/website/www/site/content/en/documentation/io/managed-io.md index ced0443c6954..ceec655e58e9 100644 --- a/website/www/site/content/en/documentation/io/managed-io.md +++ b/website/www/site/content/en/documentation/io/managed-io.md @@ -134,34 +134,13 @@ and Beam SQL is invoked via the Managed API under the hood. - BIGQUERY - - kms_key (str)
- query (str)
- row_restriction (str)
- fields (list[str])
- table (str)
- - - table (str)
- drop (list[str])
- keep (list[str])
- kms_key (str)
- only (str)
- triggering_frequency_seconds (int64)
- - - - POSTGRES + MYSQL jdbc_url (str)
connection_init_sql (list[str])
connection_properties (str)
disable_auto_commit (boolean)
- driver_class_name (str)
- driver_jars (str)
fetch_size (int32)
- jdbc_type (str)
location (str)
num_partitions (int32)
output_parallelization (boolean)
@@ -176,9 +155,6 @@ and Beam SQL is invoked via the Managed API under the hood. batch_size (int64)
connection_init_sql (list[str])
connection_properties (str)
- driver_class_name (str)
- driver_jars (str)
- jdbc_type (str)
location (str)
password (str)
username (str)
@@ -189,13 +165,9 @@ and Beam SQL is invoked via the Managed API under the hood. SQLSERVER jdbc_url (str)
- connection_init_sql (list[str])
connection_properties (str)
disable_auto_commit (boolean)
- driver_class_name (str)
- driver_jars (str)
fetch_size (int32)
- jdbc_type (str)
location (str)
num_partitions (int32)
output_parallelization (boolean)
@@ -208,11 +180,7 @@ and Beam SQL is invoked via the Managed API under the hood. jdbc_url (str)
autosharding (boolean)
batch_size (int64)
- connection_init_sql (list[str])
connection_properties (str)
- driver_class_name (str)
- driver_jars (str)
- jdbc_type (str)
location (str)
password (str)
username (str)
@@ -220,16 +188,29 @@ and Beam SQL is invoked via the Managed API under the hood. - MYSQL + BIGQUERY + + kms_key (str)
+ query (str)
+ row_restriction (str)
+ fields (list[str])
+ table (str)
+ + + table (str)
+ drop (list[str])
+ keep (list[str])
+ kms_key (str)
+ only (str)
+ triggering_frequency_seconds (int64)
+ + + + POSTGRES jdbc_url (str)
- connection_init_sql (list[str])
connection_properties (str)
- disable_auto_commit (boolean)
- driver_class_name (str)
- driver_jars (str)
fetch_size (int32)
- jdbc_type (str)
location (str)
num_partitions (int32)
output_parallelization (boolean)
@@ -242,11 +223,7 @@ and Beam SQL is invoked via the Managed API under the hood. jdbc_url (str)
autosharding (boolean)
batch_size (int64)
- connection_init_sql (list[str])
connection_properties (str)
- driver_class_name (str)
- driver_jars (str)
- jdbc_type (str)
location (str)
password (str)
username (str)
@@ -913,7 +890,7 @@ For more information on table properties, please visit https://iceberg.apache.or -### `BIGQUERY` Read +### `MYSQL` Read
@@ -924,141 +901,140 @@ For more information on table properties, please visit https://iceberg.apache.or -
- kms_key + jdbc_url str - Use this Cloud KMS key to encrypt your data + Connection URL for the JDBC source.
- query + connection_init_sql - str + list[str] - The SQL query to be executed to read from the BigQuery table. + Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this.
- row_restriction + connection_properties str - Read only rows that match this filter, which must be compatible with Google standard SQL. This is not supported when reading via query. + Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;".
- fields + disable_auto_commit - list[str] + boolean - Read only the specified fields (columns) from a BigQuery table. Fields may not be returned in the order specified. If no value is specified, then all fields are returned. Example: "col1, col2, col3" + Whether to disable auto commit on read. Defaults to true if not provided. The need for this config varies depending on the database platform. Informix requires this to be set to false while Postgres requires this to be set to true.
- table + fetch_size - str + int32 - The fully-qualified name of the BigQuery table to read from. Format: [${PROJECT}:]${DATASET}.${TABLE} + This method is used to override the size of the data that is going to be fetched and loaded in memory per every database call. It should ONLY be used if the default value throws memory errors.
-
- -### `BIGQUERY` Write - -
- - - - + + +
ConfigurationTypeDescription + location + + str + + Name of the table to read from. +
- table + num_partitions - str + int32 - The bigquery table to write to. Format: [${PROJECT}:]${DATASET}.${TABLE} + The number of partitions
- drop + output_parallelization - list[str] + boolean - A list of field names to drop from the input record before writing. Is mutually exclusive with 'keep' and 'only'. + Whether to reshuffle the resulting PCollection so results are distributed to all workers.
- keep + partition_column - list[str] + str - A list of field names to keep in the input record. All other fields are dropped before writing. Is mutually exclusive with 'drop' and 'only'. + Name of a column of numeric type that will be used for partitioning.
- kms_key + password str - Use this Cloud KMS key to encrypt your data + Password for the JDBC source.
- only + read_query str - The name of a single record field that should be written. Is mutually exclusive with 'keep' and 'drop'. + SQL query used to query the JDBC source.
- triggering_frequency_seconds + username - int64 + str - Determines how often to 'commit' progress into BigQuery. Default is every 5 seconds. + Username for the JDBC source.
-### `POSTGRES` Write +### `MYSQL` Write
@@ -1122,39 +1098,6 @@ For more information on table properties, please visit https://iceberg.apache.or Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;". - - - - - - - - - - - - - - -
- driver_class_name - - str - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". -
- driver_jars - - str - - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. -
- jdbc_type - - str - - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. -
location @@ -1202,7 +1145,7 @@ For more information on table properties, please visit https://iceberg.apache.or
-### `POSTGRES` Read +### `SQLSERVER` Write
@@ -1219,84 +1162,40 @@ For more information on table properties, please visit https://iceberg.apache.or str - - - - - - - - - - - - - - - - - - - - @@ -1307,40 +1206,7 @@ For more information on table properties, please visit https://iceberg.apache.or str - - - - - - - - - - - - - - - @@ -1356,24 +1222,24 @@ For more information on table properties, please visit https://iceberg.apache.or
- Connection URL for the JDBC source. -
- connection_init_sql - - list[str] - - Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this. -
- connection_properties - - str - - Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;". + Connection URL for the JDBC sink.
- disable_auto_commit + autosharding boolean - Whether to disable auto commit on read. Defaults to true if not provided. The need for this config varies depending on the database platform. Informix requires this to be set to false while Postgres requires this to be set to true. -
- driver_class_name - - str - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". -
- driver_jars - - str - - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. + If true, enables using a dynamically determined number of shards to write.
- fetch_size + batch_size - int32 + int64 - This method is used to override the size of the data that is going to be fetched and loaded in memory per every database call. It should ONLY be used if the default value throws memory errors. + n/a
- jdbc_type + connection_properties str - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. + Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;".
- Name of the table to read from. -
- num_partitions - - int32 - - The number of partitions -
- output_parallelization - - boolean - - Whether to reshuffle the resulting PCollection so results are distributed to all workers. -
- partition_column - - str - - Name of a column of numeric type that will be used for partitioning. + Name of the table to write to.
- read_query + username str - SQL query used to query the JDBC source. + Username for the JDBC source.
- username + write_statement str - Username for the JDBC source. + SQL query used to insert records into the JDBC sink.
@@ -1399,17 +1265,6 @@ For more information on table properties, please visit https://iceberg.apache.or Connection URL for the JDBC source. - - - connection_init_sql - - - list[str] - - - Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this. - - connection_properties @@ -1432,28 +1287,6 @@ For more information on table properties, please visit https://iceberg.apache.or Whether to disable auto commit on read. Defaults to true if not provided. The need for this config varies depending on the database platform. Informix requires this to be set to false while Postgres requires this to be set to true. - - - driver_class_name - - - str - - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". - - - - - driver_jars - - - str - - - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. - - fetch_size @@ -1465,17 +1298,6 @@ For more information on table properties, please visit https://iceberg.apache.or This method is used to override the size of the data that is going to be fetched and loaded in memory per every database call. It should ONLY be used if the default value throws memory errors. - - - jdbc_type - - - str - - - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. - - location @@ -1556,7 +1378,7 @@ For more information on table properties, please visit https://iceberg.apache.or
-### `SQLSERVER` Write +### `BIGQUERY` Read
@@ -1567,140 +1389,141 @@ For more information on table properties, please visit https://iceberg.apache.or +
- jdbc_url + kms_key str - Connection URL for the JDBC sink. + Use this Cloud KMS key to encrypt your data
- autosharding + query - boolean + str - If true, enables using a dynamically determined number of shards to write. + The SQL query to be executed to read from the BigQuery table.
- batch_size + row_restriction - int64 + str - n/a + Read only rows that match this filter, which must be compatible with Google standard SQL. This is not supported when reading via query.
- connection_init_sql + fields list[str] - Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this. + Read only the specified fields (columns) from a BigQuery table. Fields may not be returned in the order specified. If no value is specified, then all fields are returned. Example: "col1, col2, col3"
- connection_properties + table str - Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;". + The fully-qualified name of the BigQuery table to read from. Format: [${PROJECT}:]${DATASET}.${TABLE}
+
+ +### `BIGQUERY` Write + +
+ - - - + + +
- driver_class_name - - str - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". - ConfigurationTypeDescription
- driver_jars + table str - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. + The bigquery table to write to. Format: [${PROJECT}:]${DATASET}.${TABLE}
- jdbc_type + drop - str + list[str] - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. + A list of field names to drop from the input record before writing. Is mutually exclusive with 'keep' and 'only'.
- location + keep - str + list[str] - Name of the table to write to. + A list of field names to keep in the input record. All other fields are dropped before writing. Is mutually exclusive with 'drop' and 'only'.
- password + kms_key str - Password for the JDBC source. + Use this Cloud KMS key to encrypt your data
- username + only str - Username for the JDBC source. + The name of a single record field that should be written. Is mutually exclusive with 'keep' and 'drop'.
- write_statement + triggering_frequency_seconds - str + int64 - SQL query used to insert records into the JDBC sink. + Determines how often to 'commit' progress into BigQuery. Default is every 5 seconds.
-### `MYSQL` Read +### `POSTGRES` Read
@@ -1720,17 +1543,6 @@ For more information on table properties, please visit https://iceberg.apache.or Connection URL for the JDBC source. - - - - - - - - - - - - - - - - - - - - - - - - -
- connection_init_sql - - list[str] - - Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this. -
connection_properties @@ -1742,39 +1554,6 @@ For more information on table properties, please visit https://iceberg.apache.or Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;".
- disable_auto_commit - - boolean - - Whether to disable auto commit on read. Defaults to true if not provided. The need for this config varies depending on the database platform. Informix requires this to be set to false while Postgres requires this to be set to true. -
- driver_class_name - - str - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". -
- driver_jars - - str - - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. -
fetch_size @@ -1786,17 +1565,6 @@ For more information on table properties, please visit https://iceberg.apache.or This method is used to override the size of the data that is going to be fetched and loaded in memory per every database call. It should ONLY be used if the default value throws memory errors.
- jdbc_type - - str - - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. -
location @@ -1877,7 +1645,7 @@ For more information on table properties, please visit https://iceberg.apache.or
-### `MYSQL` Write +### `POSTGRES` Write
@@ -1919,17 +1687,6 @@ For more information on table properties, please visit https://iceberg.apache.or n/a - - - - - - - - - - - - - - - - - - - -
- connection_init_sql - - list[str] - - Sets the connection init sql statements used by the Driver. Only MySQL and MariaDB support this. -
connection_properties @@ -1941,39 +1698,6 @@ For more information on table properties, please visit https://iceberg.apache.or Used to set connection properties passed to the JDBC driver not already defined as standalone parameter (e.g. username and password can be set using parameters above accordingly). Format of the string must be "key1=value1;key2=value2;".
- driver_class_name - - str - - Name of a Java Driver class to use to connect to the JDBC source. For example, "com.mysql.jdbc.Driver". -
- driver_jars - - str - - Comma separated path(s) for the JDBC driver jar(s). This can be a local path or GCS (gs://) path. -
- jdbc_type - - str - - Type of JDBC source. When specified, an appropriate default Driver will be packaged with the transform. One of mysql, postgres, oracle, or mssql. -
location