diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bdf0b3a85..7c02958856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ #### New Features +- Added support for targeted delete-insert via the `overwrite_condition` parameter in `DataFrameWriter.save_as_table` + #### Bug Fixes #### Improvements diff --git a/src/snowflake/snowpark/_internal/analyzer/analyzer.py b/src/snowflake/snowpark/_internal/analyzer/analyzer.py index fe939a5b14..39d21f2e02 100644 --- a/src/snowflake/snowpark/_internal/analyzer/analyzer.py +++ b/src/snowflake/snowpark/_internal/analyzer/analyzer.py @@ -1228,6 +1228,12 @@ def do_resolve_with_resolved_children( child_attributes=resolved_child.attributes, iceberg_config=iceberg_config, table_exists=logical_plan.table_exists, + overwrite_condition=self.analyze( + logical_plan.overwrite_condition, + df_aliased_col_name_to_real_col_name, + ) + if logical_plan.overwrite_condition + else None, ) if isinstance(logical_plan, Limit): diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py index a3d40995a7..b979cd38c9 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan.py @@ -1267,6 +1267,7 @@ def save_as_table( child_attributes: Optional[List[Attribute]], iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, + overwrite_condition: Optional[str] = None, ) -> SnowflakePlan: """Returns a SnowflakePlan to materialize the child plan into a table. @@ -1417,6 +1418,47 @@ def get_create_and_insert_plan(child: SnowflakePlan, replace, error): referenced_ctes=child.referenced_ctes, ) + def get_overwrite_delete_insert_plan(child: SnowflakePlan): + """Build a plan for targeted delete + insert with transaction. + + Deletes rows matching the overwrite_condition condition, then inserts + all rows from the source DataFrame. Wrapped in a transaction for atomicity. + """ + child = self.add_result_scan_if_not_select(child) + + return SnowflakePlan( + [ + *child.queries[:-1], + Query("BEGIN TRANSACTION"), + Query( + delete_statement( + table_name=full_table_name, + condition=overwrite_condition, + source_data=None, + ), + params=child.queries[-1].params, + is_ddl_on_temp_object=is_temp_table_type, + ), + Query( + insert_into_statement( + table_name=full_table_name, + child=child.queries[-1].sql, + column_names=column_names, + ), + params=child.queries[-1].params, + is_ddl_on_temp_object=is_temp_table_type, + ), + Query("COMMIT"), + ], + schema_query=None, + post_actions=child.post_actions, + expr_to_alias={}, + source_plan=source_plan, + api_calls=child.api_calls, + session=self.session, + referenced_ctes=child.referenced_ctes, + ) + if mode == SaveMode.APPEND: assert table_exists is not None if table_exists: @@ -1446,7 +1488,12 @@ def get_create_and_insert_plan(child: SnowflakePlan, replace, error): return get_create_table_as_select_plan(child, replace=True, error=True) elif mode == SaveMode.OVERWRITE: - return get_create_table_as_select_plan(child, replace=True, error=True) + if overwrite_condition is not None and table_exists: + # Selective overwrite: delete matching rows, then insert + return get_overwrite_delete_insert_plan(child) + else: + # Default overwrite: drop and recreate table + return get_create_table_as_select_plan(child, replace=True, error=True) elif mode == SaveMode.IGNORE: return get_create_table_as_select_plan(child, replace=False, error=False) diff --git a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py index ff43b07cb2..0d2169e9eb 100644 --- a/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py +++ b/src/snowflake/snowpark/_internal/analyzer/snowflake_plan_node.py @@ -243,6 +243,7 @@ def __init__( copy_grants: bool = False, iceberg_config: Optional[dict] = None, table_exists: Optional[bool] = None, + overwrite_condition: Optional[Expression] = None, ) -> None: super().__init__() @@ -267,6 +268,7 @@ def __init__( # whether the table already exists in the database # determines the compiled SQL for APPEND and TRUNCATE mode self.table_exists = table_exists + self.overwrite_condition = overwrite_condition @property def individual_node_complexity(self) -> Dict[PlanNodeCategory, int]: diff --git a/src/snowflake/snowpark/_internal/proto/ast.proto b/src/snowflake/snowpark/_internal/proto/ast.proto index 9d32c71d7c..b3f792d4c4 100644 --- a/src/snowflake/snowpark/_internal/proto/ast.proto +++ b/src/snowflake/snowpark/_internal/proto/ast.proto @@ -1,5 +1,5 @@ // N.B. This file is generated by `//ir-dsl-c`. DO NOT EDIT! -// Generated from `{git@github.com:snowflakedb/snowflake.git}/Snowpark/ast`. +// Generated from `{git@github.com:snowflake-eng/snowflake.git}/Snowpark/ast`. syntax = "proto3"; @@ -987,7 +987,7 @@ message DataframeCollect { repeated Tuple_String_String statement_params = 7; } -// dataframe-io.ir:165 +// dataframe-io.ir:167 message DataframeCopyIntoTable { repeated Tuple_String_Expr copy_options = 1; Expr df = 2; @@ -1011,7 +1011,7 @@ message DataframeCount { repeated Tuple_String_String statement_params = 4; } -// dataframe-io.ir:148 +// dataframe-io.ir:150 message DataframeCreateOrReplaceDynamicTable { repeated Expr clustering_keys = 1; google.protobuf.StringValue comment = 2; @@ -1030,7 +1030,7 @@ message DataframeCreateOrReplaceDynamicTable { string warehouse = 15; } -// dataframe-io.ir:139 +// dataframe-io.ir:141 message DataframeCreateOrReplaceView { google.protobuf.StringValue comment = 1; bool copy_grants = 2; @@ -2685,7 +2685,7 @@ message WindowSpecRowsBetween { WindowSpecExpr wnd = 4; } -// dataframe-io.ir:116 +// dataframe-io.ir:118 message WriteCopyIntoLocation { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2704,7 +2704,7 @@ message WriteCopyIntoLocation { Expr writer = 15; } -// dataframe-io.ir:123 +// dataframe-io.ir:125 message WriteCsv { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2731,7 +2731,7 @@ message WriteFile { } } -// dataframe-io.ir:129 +// dataframe-io.ir:131 message WriteInsertInto { bool overwrite = 1; SrcPosition src = 2; @@ -2739,7 +2739,7 @@ message WriteInsertInto { Expr writer = 4; } -// dataframe-io.ir:125 +// dataframe-io.ir:127 message WriteJson { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2773,7 +2773,7 @@ message WritePandas { string table_type = 13; } -// dataframe-io.ir:127 +// dataframe-io.ir:129 message WriteParquet { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2790,7 +2790,7 @@ message WriteParquet { Expr writer = 13; } -// dataframe-io.ir:121 +// dataframe-io.ir:123 message WriteSave { bool block = 1; repeated Tuple_String_Expr copy_options = 2; @@ -2821,9 +2821,11 @@ message WriteTable { repeated Tuple_String_Expr iceberg_config = 10; google.protobuf.Int64Value max_data_extension_time = 11; SaveMode mode = 12; - SrcPosition src = 13; - repeated Tuple_String_String statement_params = 14; - NameRef table_name = 15; - string table_type = 16; - Expr writer = 17; + Expr overwrite_condition = 13; + SrcPosition src = 14; + repeated Tuple_String_String statement_params = 15; + google.protobuf.BoolValue table_exists = 16; + NameRef table_name = 17; + string table_type = 18; + Expr writer = 19; } diff --git a/src/snowflake/snowpark/dataframe_writer.py b/src/snowflake/snowpark/dataframe_writer.py index 4f272da557..c1d2c4da41 100644 --- a/src/snowflake/snowpark/dataframe_writer.py +++ b/src/snowflake/snowpark/dataframe_writer.py @@ -48,7 +48,7 @@ warning, ) from snowflake.snowpark.async_job import AsyncJob, _AsyncResultType -from snowflake.snowpark.column import Column, _to_col_if_str +from snowflake.snowpark.column import Column, _to_col_if_str, _to_col_if_sql_expr from snowflake.snowpark.exceptions import SnowparkClientException from snowflake.snowpark.functions import sql_expr from snowflake.snowpark.mock._connection import MockServerConnection @@ -256,6 +256,7 @@ def save_as_table( Dict[str, Union[str, Iterable[ColumnOrSqlExpr]]] ] = None, table_exists: Optional[bool] = None, + overwrite_condition: Optional[ColumnOrSqlExpr] = None, _emit_ast: bool = True, **kwargs: Optional[Dict[str, Any]], ) -> Optional[AsyncJob]: @@ -270,7 +271,9 @@ def save_as_table( "append": Append data of this DataFrame to the existing table. Creates a table if it does not exist. - "overwrite": Overwrite the existing table by dropping old table. + "overwrite": Overwrite the existing table. By default, drops and recreates the table. + When ``overwrite_condition`` is specified, performs selective overwrite: deletes only + rows matching the condition, then inserts new data. "truncate": Overwrite the existing table by truncating old table. @@ -330,7 +333,12 @@ def save_as_table( * iceberg_version: Overrides the version of iceberg to use. Defaults to 2 when unset. table_exists: Optional parameter to specify if the table is known to exist or not. Set to ``True`` if table exists, ``False`` if it doesn't, or ``None`` (default) for automatic detection. - Primarily useful for "append" and "truncate" modes to avoid running query for automatic detection. + Primarily useful for "append", "truncate", and "overwrite" with overwrite_condition modes to avoid running query for automatic detection. + overwrite_condition: Specifies the overwrite condition to perform atomic targeted delete-insert. + Can only be used when ``mode`` is "overwrite". When provided and the table exists, rows matching + the condition are atomically deleted and all rows from the DataFrame are inserted, preserving + non-matching rows. When not provided, the default "overwrite" behavior applies (drop and recreate table). + If the table does not exist, ``overwrite_condition`` is ignored and the table is created normally. Example 1:: @@ -364,6 +372,21 @@ def save_as_table( ... "partition_by": ["a", bucket(3, col("b"))], ... } >>> df.write.mode("overwrite").save_as_table("my_table", iceberg_config=iceberg_config) # doctest: +SKIP + + Example 3:: + + Using overwrite_condition for targeted delete and insert: + + >>> from snowflake.snowpark.functions import col + >>> df = session.create_dataframe([[1, "a"], [2, "b"], [3, "c"]], schema=["id", "val"]) + >>> df.write.mode("overwrite").save_as_table("my_table", table_type="temporary") + >>> session.table("my_table").order_by("id").collect() + [Row(ID=1, VAL='a'), Row(ID=2, VAL='b'), Row(ID=3, VAL='c')] + + >>> new_df = session.create_dataframe([[2, "updated2"], [5, "updated5"]], schema=["id", "val"]) + >>> new_df.write.mode("overwrite").save_as_table("my_table", overwrite_condition="id = 1 or val = 'b'") + >>> session.table("my_table").order_by("id").collect() + [Row(ID=2, VAL='updated2'), Row(ID=3, VAL='c'), Row(ID=5, VAL='updated5')] """ statement_params = track_data_source_statement_params( @@ -392,6 +415,8 @@ def save_as_table( # change_tracking: Optional[bool] = None, # copy_grants: bool = False, # iceberg_config: Optional[dict] = None, + # table_exists: Optional[bool] = None, + # overwrite_condition: Optional[ColumnOrSqlExpr] = None, build_table_name(expr.table_name, table_name) @@ -433,6 +458,12 @@ def save_as_table( t = expr.iceberg_config.add() t._1 = k build_expr_from_python_val(t._2, v) + if table_exists is not None: + expr.table_exists.value = table_exists + if overwrite_condition is not None: + build_expr_from_snowpark_column_or_sql_str( + expr.overwrite_condition, overwrite_condition + ) self._dataframe._session._ast_batch.eval(stmt) @@ -486,18 +517,33 @@ def save_as_table( f"Unsupported table type. Expected table types: {SUPPORTED_TABLE_TYPES}" ) + # overwrite_condition must be used with OVERWRITE mode only + if overwrite_condition is not None and save_mode != SaveMode.OVERWRITE: + raise ValueError( + f"'overwrite_condition' is only supported with mode='overwrite'. " + f"Got mode='{save_mode.value}'." + ) + + overwrite_condition_expr = ( + _to_col_if_sql_expr( + overwrite_condition, "DataFrameWriter.save_as_table" + )._expression + if overwrite_condition is not None + else None + ) + session = self._dataframe._session + needs_table_exists_check = save_mode in [ + SaveMode.APPEND, + SaveMode.TRUNCATE, + ] or (save_mode == SaveMode.OVERWRITE and overwrite_condition is not None) if ( table_exists is None and not isinstance(session._conn, MockServerConnection) - and save_mode - in [ - SaveMode.APPEND, - SaveMode.TRUNCATE, - ] + and needs_table_exists_check ): # whether the table already exists in the database - # determines the compiled SQL for APPEND and TRUNCATE mode + # determines the compiled SQL for APPEND, TRUNCATE, and OVERWRITE with overwrite_condition # if the table does not exist, we need to create it first; # if the table exists, we can skip the creation step and insert data directly table_exists = session._table_exists(table_name) @@ -518,6 +564,7 @@ def save_as_table( copy_grants, iceberg_config, table_exists, + overwrite_condition_expr, ) snowflake_plan = session._analyzer.resolve(create_table_logic_plan) result = session._conn.execute( diff --git a/tests/ast/data/DataFrame.write.test b/tests/ast/data/DataFrame.write.test index 4a67d868fc..6590c3f162 100644 --- a/tests/ast/data/DataFrame.write.test +++ b/tests/ast/data/DataFrame.write.test @@ -24,6 +24,10 @@ df.write.mode("overwrite").save_as_table("iceberg_table_target_size", iceberg_co df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={"external_volume": "example_volume", "partition_by": [bucket(10, "A"), year("date_col")], "target_file_size": "AUTO", "catalog": "my_catalog"}) +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition=(col("A") > 4) | (col("B") == "c")) + +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition="A = 1 or B = 'b'") + df.write.mode("truncate").save_as_table("test_destination", column_order="name", create_temp_table=False, table_type="transient", clustering_keys=['STR', col('num1')], comment="test", block=True, statement_params={"k":"v"}) df.write.partition_by("value").mode("overwrite").save_as_table("saved_table", table_type="temporary") @@ -130,6 +134,14 @@ df.write.mode("overwrite").save_as_table("iceberg_table_full", iceberg_config={" df = session.table("table1") +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition=(col("A") > 4) | (col("B") == "c")) + +df = session.table("table1") + +df.write.mode("overwrite").save_as_table("saved_table", table_type="temporary", table_exists=True, overwrite_condition="A = 1 or B = 'b'") + +df = session.table("table1") + df.write.mode("truncate").save_as_table("test_destination", column_order="name", table_type="transient", clustering_keys=["STR", col("num1")], statement_params={"k": "v"}, comment="test") df = session.table("table1") @@ -1849,6 +1861,315 @@ body { uid: 1 } } +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + or { + lhs { + gt { + lhs { + apply_expr { + fn { + builtin_fn { + name { + name { + name_flat { + name: "col" + } + } + } + } + } + pos_args { + string_val { + src { + end_column: 136 + end_line: 49 + file: 2 + start_column: 128 + start_line: 49 + } + v: "A" + } + } + src { + end_column: 136 + end_line: 49 + file: 2 + start_column: 128 + start_line: 49 + } + } + } + rhs { + int64_val { + src { + end_column: 140 + end_line: 49 + file: 2 + start_column: 128 + start_line: 49 + } + v: 4 + } + } + src { + end_column: 140 + end_line: 49 + file: 2 + start_column: 128 + start_line: 49 + } + } + } + rhs { + eq { + lhs { + apply_expr { + fn { + builtin_fn { + name { + name { + name_flat { + name: "col" + } + } + } + } + } + pos_args { + string_val { + src { + end_column: 153 + end_line: 49 + file: 2 + start_column: 145 + start_line: 49 + } + v: "B" + } + } + src { + end_column: 153 + end_line: 49 + file: 2 + start_column: 145 + start_line: 49 + } + } + } + rhs { + string_val { + src { + end_column: 160 + end_line: 49 + file: 2 + start_column: 145 + start_line: 49 + } + v: "c" + } + } + src { + end_column: 160 + end_line: 49 + file: 2 + start_column: 145 + start_line: 49 + } + } + } + src { + end_column: 161 + end_line: 49 + file: 2 + start_column: 127 + start_line: 49 + } + } + } + src { + end_column: 162 + end_line: 49 + file: 2 + start_column: 8 + start_line: 49 + } + table_exists { + value: true + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_overwrite: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 12 + } +} +body { + eval { + bind_id: 12 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} +body { + bind { + expr { + write_table { + block: true + column_order: "index" + overwrite_condition { + sql_expr { + sql: "A = 1 or B = \'b\'" + src { + end_column: 146 + end_line: 51 + file: 2 + start_column: 8 + start_line: 51 + } + } + } + src { + end_column: 146 + end_line: 51 + file: 2 + start_column: 8 + start_line: 51 + } + table_exists { + value: true + } + table_name { + name { + name_flat { + name: "saved_table" + } + } + } + table_type: "temporary" + writer { + dataframe_writer { + df { + dataframe_ref { + id: 1 + } + } + save_mode { + save_mode_overwrite: true + } + src { + end_column: 16 + end_line: 29 + file: 2 + start_column: 8 + start_line: 29 + } + } + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + } + uid: 13 + } +} +body { + eval { + bind_id: 13 + } +} +body { + bind { + expr { + table { + name { + name { + name_flat { + name: "table1" + } + } + } + src { + end_column: 41 + end_line: 27 + file: 2 + start_column: 13 + start_line: 27 + } + variant { + session_table: true + } + } + } + first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" + symbol { + value: "df" + } + uid: 1 + } +} body { bind { expr { @@ -1858,10 +2179,10 @@ body { string_val { src { end_column: 231 - end_line: 49 + end_line: 53 file: 2 start_column: 8 - start_line: 49 + start_line: 53 } v: "STR" } @@ -1883,20 +2204,20 @@ body { string_val { src { end_column: 173 - end_line: 49 + end_line: 53 file: 2 start_column: 162 - start_line: 49 + start_line: 53 } v: "num1" } } src { end_column: 173 - end_line: 49 + end_line: 53 file: 2 start_column: 162 - start_line: 49 + start_line: 53 } } } @@ -1906,10 +2227,10 @@ body { } src { end_column: 231 - end_line: 49 + end_line: 53 file: 2 start_column: 8 - start_line: 49 + start_line: 53 } statement_params { _1: "k" @@ -1947,12 +2268,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 12 + uid: 14 } } body { eval { - bind_id: 12 + bind_id: 14 } } body { @@ -1993,10 +2314,10 @@ body { column_order: "index" src { end_column: 109 - end_line: 51 + end_line: 55 file: 2 start_column: 8 - start_line: 51 + start_line: 55 } table_name { name { @@ -2018,10 +2339,10 @@ body { sql: "value" src { end_column: 38 - end_line: 51 + end_line: 55 file: 2 start_column: 8 - start_line: 51 + start_line: 55 } } } @@ -2042,12 +2363,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 13 + uid: 15 } } body { eval { - bind_id: 13 + bind_id: 15 } } body { @@ -2088,10 +2409,10 @@ body { column_order: "index" src { end_column: 114 - end_line: 53 + end_line: 57 file: 2 start_column: 8 - start_line: 53 + start_line: 57 } table_name { name { @@ -2125,20 +2446,20 @@ body { string_val { src { end_column: 42 - end_line: 53 + end_line: 57 file: 2 start_column: 30 - start_line: 53 + start_line: 57 } v: "value" } } src { end_column: 42 - end_line: 53 + end_line: 57 file: 2 start_column: 30 - start_line: 53 + start_line: 57 } } } @@ -2159,12 +2480,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 14 + uid: 16 } } body { eval { - bind_id: 14 + bind_id: 16 } } body { @@ -2205,10 +2526,10 @@ body { column_order: "index" src { end_column: 151 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } table_name { name { @@ -2231,10 +2552,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -2257,20 +2578,20 @@ body { string_val { src { end_column: 79 - end_line: 55 + end_line: 59 file: 2 start_column: 67 - start_line: 55 + start_line: 59 } v: "value" } } src { end_column: 79 - end_line: 55 + end_line: 59 file: 2 start_column: 67 - start_line: 55 + start_line: 59 } } } @@ -2291,12 +2612,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 15 + uid: 17 } } body { eval { - bind_id: 15 + bind_id: 17 } } body { @@ -2337,10 +2658,10 @@ body { column_order: "index" src { end_column: 239 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } table_name { name { @@ -2363,10 +2684,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -2378,10 +2699,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -2393,10 +2714,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -2408,10 +2729,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -2423,10 +2744,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -2449,20 +2770,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -2483,12 +2804,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 16 + uid: 18 } } body { eval { - bind_id: 16 + bind_id: 18 } } body { @@ -2498,10 +2819,10 @@ body { query: "create temp stage if not exists test_stage" src { end_column: 88 - end_line: 59 + end_line: 63 file: 2 start_column: 31 - start_line: 59 + start_line: 63 } } } @@ -2509,7 +2830,7 @@ body { symbol { value: "stage_created_result" } - uid: 17 + uid: 19 } } body { @@ -2520,27 +2841,27 @@ body { case_sensitive: true df { dataframe_ref { - id: 17 + id: 19 } } src { end_column: 98 - end_line: 59 + end_line: 63 file: 2 start_column: 31 - start_line: 59 + start_line: 63 } } } first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 18 + uid: 20 } } body { eval { - bind_id: 18 + bind_id: 20 } } body { @@ -2581,10 +2902,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 72 - end_line: 61 + end_line: 65 file: 2 start_column: 8 - start_line: 61 + start_line: 65 } writer { dataframe_writer { @@ -2599,10 +2920,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -2614,10 +2935,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -2629,10 +2950,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -2644,10 +2965,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -2659,10 +2980,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -2685,20 +3006,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -2719,12 +3040,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 19 + uid: 21 } } body { eval { - bind_id: 19 + bind_id: 21 } } body { @@ -2768,10 +3089,10 @@ body { bool_val { src { end_column: 123 - end_line: 65 + end_line: 69 file: 2 start_column: 8 - start_line: 65 + start_line: 69 } v: true } @@ -2783,10 +3104,10 @@ body { bool_val { src { end_column: 123 - end_line: 65 + end_line: 69 file: 2 start_column: 8 - start_line: 65 + start_line: 69 } v: true } @@ -2799,10 +3120,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 123 - end_line: 65 + end_line: 69 file: 2 start_column: 8 - start_line: 65 + start_line: 69 } writer { dataframe_writer { @@ -2817,10 +3138,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -2832,10 +3153,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -2847,10 +3168,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -2862,10 +3183,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -2877,10 +3198,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -2903,20 +3224,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -2937,12 +3258,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 20 + uid: 22 } } body { eval { - bind_id: 20 + bind_id: 22 } } body { @@ -2985,10 +3306,10 @@ body { bool_val { src { end_column: 181 - end_line: 67 + end_line: 71 file: 2 start_column: 8 - start_line: 67 + start_line: 71 } v: true } @@ -3000,10 +3321,10 @@ body { bool_val { src { end_column: 181 - end_line: 67 + end_line: 71 file: 2 start_column: 8 - start_line: 67 + start_line: 71 } } } @@ -3018,10 +3339,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 181 - end_line: 67 + end_line: 71 file: 2 start_column: 8 - start_line: 67 + start_line: 71 } writer { dataframe_writer { @@ -3036,10 +3357,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -3051,10 +3372,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -3066,10 +3387,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -3081,10 +3402,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -3096,10 +3417,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -3122,20 +3443,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -3156,12 +3477,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 21 + uid: 23 } } body { eval { - bind_id: 21 + bind_id: 23 } } body { @@ -3204,10 +3525,10 @@ body { bool_val { src { end_column: 207 - end_line: 69 + end_line: 73 file: 2 start_column: 8 - start_line: 69 + start_line: 73 } v: true } @@ -3219,10 +3540,10 @@ body { bool_val { src { end_column: 207 - end_line: 69 + end_line: 73 file: 2 start_column: 8 - start_line: 69 + start_line: 73 } } } @@ -3241,10 +3562,10 @@ body { location: "@test_stage/copied_from_dataframe" src { end_column: 207 - end_line: 69 + end_line: 73 file: 2 start_column: 8 - start_line: 69 + start_line: 73 } writer { dataframe_writer { @@ -3259,10 +3580,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -3274,10 +3595,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -3289,10 +3610,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -3304,10 +3625,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -3319,10 +3640,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -3345,20 +3666,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -3379,12 +3700,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 22 + uid: 24 } } body { eval { - bind_id: 22 + bind_id: 24 } } body { @@ -3428,10 +3749,10 @@ body { bool_val { src { end_column: 125 - end_line: 73 + end_line: 77 file: 2 start_column: 8 - start_line: 73 + start_line: 77 } v: true } @@ -3443,10 +3764,10 @@ body { bool_val { src { end_column: 125 - end_line: 73 + end_line: 77 file: 2 start_column: 8 - start_line: 73 + start_line: 77 } v: true } @@ -3460,10 +3781,10 @@ body { location: "@test_stage/test.csv" src { end_column: 125 - end_line: 73 + end_line: 77 file: 2 start_column: 8 - start_line: 73 + start_line: 77 } writer { dataframe_writer { @@ -3478,10 +3799,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -3493,10 +3814,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -3508,10 +3829,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -3523,10 +3844,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -3538,10 +3859,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -3564,20 +3885,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -3598,12 +3919,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 23 + uid: 25 } } body { eval { - bind_id: 23 + bind_id: 25 } } body { @@ -3648,10 +3969,10 @@ body { location: "@test_stage/test.csv" src { end_column: 139 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } writer { dataframe_writer { @@ -3669,10 +3990,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -3684,10 +4005,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -3699,10 +4020,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -3714,10 +4035,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -3729,10 +4050,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -3744,10 +4065,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -3770,20 +4091,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -3804,12 +4125,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 24 + uid: 26 } } body { eval { - bind_id: 24 + bind_id: 26 } } body { @@ -3853,10 +4174,10 @@ body { bool_val { src { end_column: 114 - end_line: 79 + end_line: 83 file: 2 start_column: 8 - start_line: 79 + start_line: 83 } v: true } @@ -3868,10 +4189,10 @@ body { bool_val { src { end_column: 114 - end_line: 79 + end_line: 83 file: 2 start_column: 8 - start_line: 79 + start_line: 83 } v: true } @@ -3884,10 +4205,10 @@ body { location: "@test_stage/test.json" src { end_column: 114 - end_line: 79 + end_line: 83 file: 2 start_column: 8 - start_line: 79 + start_line: 83 } writer { dataframe_writer { @@ -3905,10 +4226,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -3920,10 +4241,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -3935,10 +4256,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -3950,10 +4271,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -3965,10 +4286,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -3980,10 +4301,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -4006,20 +4327,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -4040,12 +4361,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 25 + uid: 27 } } body { eval { - bind_id: 25 + bind_id: 27 } } body { @@ -4090,10 +4411,10 @@ body { location: "@test_stage/test.json" src { end_column: 117 - end_line: 81 + end_line: 85 file: 2 start_column: 8 - start_line: 81 + start_line: 85 } writer { dataframe_writer { @@ -4111,10 +4432,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -4126,10 +4447,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -4141,10 +4462,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -4156,10 +4477,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -4171,10 +4492,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -4186,10 +4507,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -4212,20 +4533,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -4246,12 +4567,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 26 + uid: 28 } } body { eval { - bind_id: 26 + bind_id: 28 } } body { @@ -4295,10 +4616,10 @@ body { bool_val { src { end_column: 130 - end_line: 85 + end_line: 89 file: 2 start_column: 8 - start_line: 85 + start_line: 89 } v: true } @@ -4310,10 +4631,10 @@ body { bool_val { src { end_column: 130 - end_line: 85 + end_line: 89 file: 2 start_column: 8 - start_line: 85 + start_line: 89 } v: true } @@ -4326,10 +4647,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 130 - end_line: 85 + end_line: 89 file: 2 start_column: 8 - start_line: 85 + start_line: 89 } writer { dataframe_writer { @@ -4347,10 +4668,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -4362,10 +4683,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -4377,10 +4698,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -4392,10 +4713,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -4407,10 +4728,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -4422,10 +4743,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -4448,20 +4769,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -4482,12 +4803,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 27 + uid: 29 } } body { eval { - bind_id: 27 + bind_id: 29 } } body { @@ -4532,10 +4853,10 @@ body { location: "@test_stage/test.parquet" src { end_column: 120 - end_line: 87 + end_line: 91 file: 2 start_column: 8 - start_line: 87 + start_line: 91 } writer { dataframe_writer { @@ -4553,10 +4874,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -4568,10 +4889,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -4583,10 +4904,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -4598,10 +4919,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -4613,10 +4934,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -4628,10 +4949,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -4654,20 +4975,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -4688,12 +5009,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 28 + uid: 30 } } body { eval { - bind_id: 28 + bind_id: 30 } } body { @@ -4733,10 +5054,10 @@ body { overwrite: true src { end_column: 59 - end_line: 89 + end_line: 93 file: 2 start_column: 8 - start_line: 89 + start_line: 93 } table_name { name { @@ -4761,10 +5082,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -4776,10 +5097,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -4791,10 +5112,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -4806,10 +5127,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -4821,10 +5142,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -4836,10 +5157,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -4862,20 +5183,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -4896,12 +5217,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 29 + uid: 31 } } body { eval { - bind_id: 29 + bind_id: 31 } } body { @@ -4940,10 +5261,10 @@ body { write_insert_into { src { end_column: 60 - end_line: 91 + end_line: 95 file: 2 start_column: 8 - start_line: 91 + start_line: 95 } table_name { name { @@ -4968,10 +5289,10 @@ body { string_val { src { end_column: 53 - end_line: 55 + end_line: 59 file: 2 start_column: 8 - start_line: 55 + start_line: 59 } v: "abcd" } @@ -4983,10 +5304,10 @@ body { string_val { src { end_column: 53 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "abcd" } @@ -4998,10 +5319,10 @@ body { string_val { src { end_column: 90 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "pqrs" } @@ -5013,10 +5334,10 @@ body { bool_val { src { end_column: 116 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: true } @@ -5028,10 +5349,10 @@ body { string_val { src { end_column: 141 - end_line: 57 + end_line: 61 file: 2 start_column: 8 - start_line: 57 + start_line: 61 } v: "append" } @@ -5043,10 +5364,10 @@ body { bool_val { src { end_column: 53 - end_line: 75 + end_line: 79 file: 2 start_column: 8 - start_line: 75 + start_line: 79 } v: true } @@ -5069,20 +5390,20 @@ body { string_val { src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } v: "value" } } src { end_column: 167 - end_line: 57 + end_line: 61 file: 2 start_column: 155 - start_line: 57 + start_line: 61 } } } @@ -5103,12 +5424,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 30 + uid: 32 } } body { eval { - bind_id: 30 + bind_id: 32 } } body { @@ -5124,10 +5445,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5138,7 +5459,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5149,10 +5470,10 @@ body { location: "s3://testbucket" src { end_column: 94 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } storage_integration { value: "test_integration" @@ -5161,15 +5482,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5178,12 +5499,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 32 + uid: 34 } } body { eval { - bind_id: 32 + bind_id: 34 } } body { @@ -5199,10 +5520,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5213,7 +5534,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5224,10 +5545,10 @@ body { location: "azure://testcontainer" src { end_column: 131 - end_line: 97 + end_line: 101 file: 2 start_column: 8 - start_line: 97 + start_line: 101 } storage_integration { value: "test_integration" @@ -5239,15 +5560,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5256,12 +5577,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 33 + uid: 35 } } body { eval { - bind_id: 33 + bind_id: 35 } } body { @@ -5277,10 +5598,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5291,7 +5612,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5302,24 +5623,24 @@ body { location: "gcp://teststorage" src { end_column: 87 - end_line: 99 + end_line: 103 file: 2 start_column: 8 - start_line: 99 + start_line: 103 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5328,12 +5649,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 34 + uid: 36 } } body { eval { - bind_id: 34 + bind_id: 36 } } body { @@ -5349,10 +5670,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5363,7 +5684,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5377,10 +5698,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 105 file: 2 start_column: 8 - start_line: 101 + start_line: 105 } v: "awsid" } @@ -5392,10 +5713,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 105 file: 2 start_column: 8 - start_line: 101 + start_line: 105 } v: "!23@" } @@ -5407,10 +5728,10 @@ body { null_val { src { end_column: 174 - end_line: 101 + end_line: 105 file: 2 start_column: 8 - start_line: 101 + start_line: 105 } } } @@ -5421,10 +5742,10 @@ body { string_val { src { end_column: 174 - end_line: 101 + end_line: 105 file: 2 start_column: 8 - start_line: 101 + start_line: 105 } v: "master key" } @@ -5433,24 +5754,24 @@ body { location: "gcp://teststorage" src { end_column: 174 - end_line: 101 + end_line: 105 file: 2 start_column: 8 - start_line: 101 + start_line: 105 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5459,12 +5780,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 35 + uid: 37 } } body { eval { - bind_id: 35 + bind_id: 37 } } body { @@ -5480,10 +5801,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5494,7 +5815,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5505,10 +5826,10 @@ body { location: "azure://testcontainer" src { end_column: 122 - end_line: 103 + end_line: 107 file: 2 start_column: 8 - start_line: 103 + start_line: 107 } storage_integration { value: "test_integration" @@ -5520,15 +5841,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5537,12 +5858,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 36 + uid: 38 } } body { eval { - bind_id: 36 + bind_id: 38 } } body { @@ -5558,10 +5879,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5572,7 +5893,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5586,10 +5907,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 109 file: 2 start_column: 8 - start_line: 105 + start_line: 109 } v: "awsid" } @@ -5601,10 +5922,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 109 file: 2 start_column: 8 - start_line: 105 + start_line: 109 } v: "!23@" } @@ -5616,10 +5937,10 @@ body { null_val { src { end_column: 158 - end_line: 105 + end_line: 109 file: 2 start_column: 8 - start_line: 105 + start_line: 109 } } } @@ -5630,10 +5951,10 @@ body { string_val { src { end_column: 158 - end_line: 105 + end_line: 109 file: 2 start_column: 8 - start_line: 105 + start_line: 109 } v: "master key" } @@ -5642,24 +5963,24 @@ body { location: "s3://testbucket" src { end_column: 158 - end_line: 105 + end_line: 109 file: 2 start_column: 8 - start_line: 105 + start_line: 109 } writer { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5668,12 +5989,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 37 + uid: 39 } } body { eval { - bind_id: 37 + bind_id: 39 } } body { @@ -5689,10 +6010,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5703,7 +6024,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5714,10 +6035,10 @@ body { location: "azure://testcontainer" src { end_column: 149 - end_line: 107 + end_line: 111 file: 2 start_column: 8 - start_line: 107 + start_line: 111 } storage_integration { value: "test_integration" @@ -5729,15 +6050,15 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5746,12 +6067,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 38 + uid: 40 } } body { eval { - bind_id: 38 + bind_id: 40 } } body { @@ -5767,10 +6088,10 @@ body { } src { end_column: 41 - end_line: 93 + end_line: 97 file: 2 start_column: 13 - start_line: 93 + start_line: 97 } variant { session_table: true @@ -5781,7 +6102,7 @@ body { symbol { value: "df" } - uid: 31 + uid: 33 } } body { @@ -5795,10 +6116,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } v: "awsid" } @@ -5810,10 +6131,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } v: "!23@" } @@ -5825,10 +6146,10 @@ body { null_val { src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } } } @@ -5839,10 +6160,10 @@ body { string_val { src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } v: "master key" } @@ -5854,19 +6175,19 @@ body { sql: "value" src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } } } src { end_column: 237 - end_line: 109 + end_line: 113 file: 2 start_column: 8 - start_line: 109 + start_line: 113 } storage_integration { value: "test_integration" @@ -5875,7 +6196,7 @@ body { dataframe_writer { df { dataframe_ref { - id: 31 + id: 33 } } format { @@ -5883,10 +6204,10 @@ body { } src { end_column: 16 - end_line: 95 + end_line: 99 file: 2 start_column: 8 - start_line: 95 + start_line: 99 } } } @@ -5895,12 +6216,12 @@ body { first_request_id: "\003U\"\366q\366P\346\260\261?\234\303\254\316\353" symbol { } - uid: 39 + uid: 41 } } body { eval { - bind_id: 39 + bind_id: 41 } } client_ast_version: 1 diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index 30e1805e5c..af24ea98e8 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -4566,6 +4566,156 @@ def test_write_table_with_clustering_keys_and_comment( Utils.drop_table(session, table_name3) +@pytest.mark.xfail( + "config.getoption('local_testing_mode', default=False)", + reason="overwrite_condition is not supported in local testing mode", + run=False, +) +def test_write_table_with_overwrite_condition(session): + """Test overwrite_condition parameter for targeted delete + insert.""" + table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) + try: + # Setup and verify initial data + initial_df = session.create_dataframe( + [[1, "a"], [2, "b"], [3, "c"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + initial_df.write.mode("overwrite").save_as_table(table_name) + result = session.table(table_name).order_by("id").collect() + assert result == [ + Row(ID=1, VAL="a"), + Row(ID=2, VAL="b"), + Row(ID=3, VAL="c"), + ] + + # Test 1: overwrite_condition with SQL string expr + new_df1 = session.create_dataframe( + [[2, "updated2"], [5, "new5"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df1.write.mode("overwrite").save_as_table( + table_name, overwrite_condition="id = 1 or val = 'b'" + ) + result = session.table(table_name).order_by("id").collect() + # id=1 and id=2 (val='b') deleted, new rows inserted + assert result == [ + Row(ID=2, VAL="updated2"), + Row(ID=3, VAL="c"), + Row(ID=5, VAL="new5"), + ] + + # Test 2: overwrite_condition with Column expr + new_df2 = session.create_dataframe( + [[2, "replaced2"], [4, "new4"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df2.write.mode("overwrite").save_as_table( + table_name, overwrite_condition=col("id") == 2 + ) + result = session.table(table_name).order_by("id").collect() + # id=2 deleted, new rows inserted + assert result == [ + Row(ID=2, VAL="replaced2"), + Row(ID=3, VAL="c"), + Row(ID=4, VAL="new4"), + Row(ID=5, VAL="new5"), + ] + + # Test 3: overwrite_condition with multiple Column expr + new_df3 = session.create_dataframe( + [[6, "new6"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df3.write.mode("overwrite").save_as_table( + table_name, overwrite_condition=(col("id") > 4) | (col("val") == "c") + ) + result = session.table(table_name).order_by("id").collect() + # id=3 (val='c') and id=5 (id > 4) deleted, id=4 remains (4 is not > 4), new row inserted + assert result == [ + Row(ID=2, VAL="replaced2"), + Row(ID=4, VAL="new4"), + Row(ID=6, VAL="new6"), + ] + + # Test 4: overwrite_condition that matches all rows + new_df4 = session.create_dataframe( + [[10, "new"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df4.write.mode("overwrite").save_as_table( + table_name, overwrite_condition="id > 0" + ) + result = session.table(table_name).collect() + assert result == [Row(ID=10, VAL="new")] + + # Test 5: overwrite_condition that matches no rows + new_df5 = session.create_dataframe( + [[20, "another"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + new_df5.write.mode("overwrite").save_as_table( + table_name, overwrite_condition="id = 999" + ) + result = session.table(table_name).order_by("id").collect() + assert result == [ + Row(ID=10, VAL="new"), + Row(ID=20, VAL="another"), + ] + + finally: + Utils.drop_table(session, table_name) + + +@pytest.mark.xfail( + "config.getoption('local_testing_mode', default=False)", + reason="overwrite_condition is a not supported in local testing mode", + run=False, +) +@pytest.mark.parametrize( + "invalid_mode", ["append", "truncate", "errorifexists", "ignore"] +) +def test_write_table_with_overwrite_condition_edge_cases(session, invalid_mode): + """Test overwrite_condition edge cases: table not exists, and invalid modes.""" + table_name = Utils.random_name_for_temp_object(TempObjectType.TABLE) + try: + # Edge case 1: Table doesn't exist - overwrite_condition is no-op + df = session.create_dataframe( + [[1, "a"], [2, "b"]], + schema=StructType( + [StructField("id", IntegerType()), StructField("val", StringType())] + ), + ) + df.write.mode("overwrite").save_as_table( + table_name, overwrite_condition="id = 999" + ) + result = session.table(table_name).order_by("id").collect() + assert result == [Row(ID=1, VAL="a"), Row(ID=2, VAL="b")] + + # Edge case 2: Invalid mode raises ValueError + with pytest.raises( + ValueError, + match="'overwrite_condition' is only supported with mode='overwrite'", + ): + df.write.mode(invalid_mode).save_as_table( + table_name, overwrite_condition="id = 1" + ) + + finally: + Utils.drop_table(session, table_name) + + @pytest.mark.xfail( "config.getoption('local_testing_mode', default=False)", reason="Clustering is a SQL feature",